2024-11-27 19:14:07,569 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-27 19:14:07,583 main DEBUG Took 0.011555 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 19:14:07,583 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 19:14:07,583 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 19:14:07,584 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 19:14:07,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,598 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 19:14:07,613 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,616 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,617 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,618 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,619 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,621 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,623 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,624 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,625 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,626 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,627 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 19:14:07,628 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,628 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 19:14:07,630 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 19:14:07,632 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 19:14:07,634 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 19:14:07,635 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 19:14:07,636 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 19:14:07,637 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 19:14:07,649 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 19:14:07,651 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 19:14:07,653 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 19:14:07,654 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 19:14:07,654 main DEBUG createAppenders(={Console}) 2024-11-27 19:14:07,655 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 initialized 2024-11-27 19:14:07,655 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 2024-11-27 19:14:07,655 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@221a3fa4 OK. 2024-11-27 19:14:07,656 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 19:14:07,656 main DEBUG OutputStream closed 2024-11-27 19:14:07,656 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 19:14:07,656 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 19:14:07,657 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3c73951 OK 2024-11-27 19:14:07,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 19:14:07,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 19:14:07,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 19:14:07,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 19:14:07,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 19:14:07,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 19:14:07,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 19:14:07,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 19:14:07,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 19:14:07,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 19:14:07,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 19:14:07,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 19:14:07,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 19:14:07,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 19:14:07,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 19:14:07,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 19:14:07,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 19:14:07,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 19:14:07,740 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 19:14:07,741 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@61fe30) with optional ClassLoader: null 2024-11-27 19:14:07,741 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 19:14:07,742 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@61fe30] started OK. 2024-11-27T19:14:07,795 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackup timeout: 13 mins 2024-11-27 19:14:07,799 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 19:14:07,800 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T19:14:08,173 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009 2024-11-27T19:14:08,179 DEBUG [Time-limited test {}] impl.BackupManager(124): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-27T19:14:08,181 DEBUG [Time-limited test {}] impl.BackupManager(154): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:14:08,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T19:14:08,204 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a, deleteOnExit=true 2024-11-27T19:14:08,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T19:14:08,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/test.cache.data in system properties and HBase conf 2024-11-27T19:14:08,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T19:14:08,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir in system properties and HBase conf 2024-11-27T19:14:08,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T19:14:08,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T19:14:08,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T19:14:08,338 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T19:14:08,456 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T19:14:08,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T19:14:08,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T19:14:08,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T19:14:08,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T19:14:08,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T19:14:08,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T19:14:08,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T19:14:08,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T19:14:08,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T19:14:08,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/nfs.dump.dir in system properties and HBase conf 2024-11-27T19:14:08,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir in system properties and HBase conf 2024-11-27T19:14:08,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T19:14:08,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T19:14:08,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T19:14:09,372 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T19:14:09,471 INFO [Time-limited test {}] log.Log(170): Logging initialized @2705ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T19:14:09,585 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:09,670 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:09,694 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:09,695 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:09,696 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:09,714 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:09,717 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:09,718 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:09,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@38e78c7b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-36109-hadoop-hdfs-3_4_1-tests_jar-_-any-11787147497797831605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T19:14:09,964 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:36109} 2024-11-27T19:14:09,964 INFO [Time-limited test {}] server.Server(415): Started @3198ms 2024-11-27T19:14:10,355 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:10,362 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:10,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:10,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:10,363 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:10,364 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:10,365 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:10,485 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@cb44ad4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-46827-hadoop-hdfs-3_4_1-tests_jar-_-any-14318491186349710267/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T19:14:10,486 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:46827} 2024-11-27T19:14:10,486 INFO [Time-limited test {}] server.Server(415): Started @3721ms 2024-11-27T19:14:10,554 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T19:14:11,072 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data2/current/BP-496223468-172.17.0.2-1732734849103/current, will proceed with Du for space computation calculation, 2024-11-27T19:14:11,072 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data1/current/BP-496223468-172.17.0.2-1732734849103/current, will proceed with Du for space computation calculation, 2024-11-27T19:14:11,117 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T19:14:11,178 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x272fded383472808 with lease ID 0xe24d92a07026eae8: Processing first storage report for DS-d5ee68fe-a069-4fae-bb51-774a35b0b379 from datanode DatanodeRegistration(127.0.0.1:45013, datanodeUuid=927051a4-f8bb-4e6a-8bb8-ec34dd96f242, infoPort=33289, infoSecurePort=0, ipcPort=36495, storageInfo=lv=-57;cid=testClusterID;nsid=97843409;c=1732734849103) 2024-11-27T19:14:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x272fded383472808 with lease ID 0xe24d92a07026eae8: from storage DS-d5ee68fe-a069-4fae-bb51-774a35b0b379 node DatanodeRegistration(127.0.0.1:45013, datanodeUuid=927051a4-f8bb-4e6a-8bb8-ec34dd96f242, infoPort=33289, infoSecurePort=0, ipcPort=36495, storageInfo=lv=-57;cid=testClusterID;nsid=97843409;c=1732734849103), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T19:14:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x272fded383472808 with lease ID 0xe24d92a07026eae8: Processing first storage report for DS-a51c57ab-fc59-445a-9b04-9bd066217584 from datanode DatanodeRegistration(127.0.0.1:45013, datanodeUuid=927051a4-f8bb-4e6a-8bb8-ec34dd96f242, infoPort=33289, infoSecurePort=0, ipcPort=36495, storageInfo=lv=-57;cid=testClusterID;nsid=97843409;c=1732734849103) 2024-11-27T19:14:11,180 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x272fded383472808 with lease ID 0xe24d92a07026eae8: from storage DS-a51c57ab-fc59-445a-9b04-9bd066217584 node DatanodeRegistration(127.0.0.1:45013, datanodeUuid=927051a4-f8bb-4e6a-8bb8-ec34dd96f242, infoPort=33289, infoSecurePort=0, ipcPort=36495, storageInfo=lv=-57;cid=testClusterID;nsid=97843409;c=1732734849103), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T19:14:11,261 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009 2024-11-27T19:14:11,359 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/zookeeper_0, clientPort=63134, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T19:14:11,372 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63134 2024-11-27T19:14:11,382 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:11,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:11,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741825_1001 (size=7) 2024-11-27T19:14:12,022 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 with version=8 2024-11-27T19:14:12,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/hbase-staging 2024-11-27T19:14:12,039 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(136): Setting Master Port to random. 2024-11-27T19:14:12,040 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(143): Setting RegionServer Port to random. 2024-11-27T19:14:12,040 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(153): Setting RS InfoServer Port to random. 2024-11-27T19:14:12,040 DEBUG [Time-limited test {}] hbase.LocalHBaseCluster(161): Setting Master InfoServer Port to random. 2024-11-27T19:14:12,137 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T19:14:12,410 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eb850308c015:0 server-side Connection retries=45 2024-11-27T19:14:12,426 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:12,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:12,434 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T19:14:12,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:12,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T19:14:12,575 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T19:14:12,641 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T19:14:12,653 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T19:14:12,658 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T19:14:12,685 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4295 (auto-detected) 2024-11-27T19:14:12,686 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T19:14:12,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39035 2024-11-27T19:14:12,728 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39035 connecting to ZooKeeper ensemble=127.0.0.1:63134 2024-11-27T19:14:12,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390350x0, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T19:14:12,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39035-0x1015ad983930000 connected 2024-11-27T19:14:12,796 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:12,799 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:12,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-27T19:14:12,846 INFO [Time-limited test {}] log.Log(170): Logging initialized @6080ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T19:14:12,906 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-27T19:14:12,906 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-27T19:14:12,907 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-27T19:14:12,907 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-27T19:14:12,908 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-27T19:14:12,910 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2024-11-27T19:14:12,910 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-27T19:14:12,910 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-27T19:14:12,912 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-27T19:14:12,999 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 46687 2024-11-27T19:14:13,001 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-27T19:14:13,022 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:13,024 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:13,025 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:13,025 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-27T19:14:13,069 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-27T19:14:13,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:13,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:13,079 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T19:14:13,083 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:13,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-27T19:14:13,783 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:46687} 2024-11-27T19:14:13,784 INFO [Time-limited test {}] server.Server(415): Started @7018ms 2024-11-27T19:14:13,787 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9, hbase.cluster.distributed=false 2024-11-27T19:14:13,812 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-27T19:14:13,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39035 2024-11-27T19:14:13,817 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39035 2024-11-27T19:14:13,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39035 2024-11-27T19:14:13,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39035 2024-11-27T19:14:13,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39035 2024-11-27T19:14:13,949 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eb850308c015:0 server-side Connection retries=45 2024-11-27T19:14:13,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:13,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:13,952 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T19:14:13,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T19:14:13,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T19:14:13,956 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T19:14:13,959 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T19:14:13,960 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41349 2024-11-27T19:14:13,962 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41349 connecting to ZooKeeper ensemble=127.0.0.1:63134 2024-11-27T19:14:13,964 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:13,968 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:13,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413490x0, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T19:14:13,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413490x0, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-27T19:14:13,979 DEBUG [Time-limited test {}] http.HttpServer(797): org.apache.hbase.thirdparty.org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker is already part of alias check list 2024-11-27T19:14:13,979 INFO [Time-limited test {}] http.HttpServer(802): Serving aliases allowed for /logs context 2024-11-27T19:14:13,980 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2024-11-27T19:14:13,980 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2024-11-27T19:14:13,980 INFO [Time-limited test {}] http.HttpServer(1023): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2024-11-27T19:14:13,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41349-0x1015ad983930001 connected 2024-11-27T19:14:13,982 INFO [Time-limited test {}] http.HttpServer(1002): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2024-11-27T19:14:13,982 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2024-11-27T19:14:13,982 INFO [Time-limited test {}] http.HttpServer(1009): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2024-11-27T19:14:13,983 INFO [Time-limited test {}] http.HttpServer(844): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2024-11-27T19:14:13,985 INFO [Time-limited test {}] http.HttpServer(1269): Jetty bound to port 44393 2024-11-27T19:14:13,985 INFO [Time-limited test {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-27T19:14:13,987 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:13,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:13,988 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:13,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2024-11-27T19:14:14,005 INFO [Time-limited test {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2024-11-27T19:14:14,006 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:14,007 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:14,007 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:14,008 INFO [Time-limited test {}] http.SecurityHeadersFilter(49): Added security headers filter 2024-11-27T19:14:14,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-27T19:14:14,013 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:44393} 2024-11-27T19:14:14,013 INFO [Time-limited test {}] server.Server(415): Started @7247ms 2024-11-27T19:14:14,017 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T19:14:14,026 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T19:14:14,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2024-11-27T19:14:14,036 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2024-11-27T19:14:14,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41349 2024-11-27T19:14:14,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41349 2024-11-27T19:14:14,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41349 2024-11-27T19:14:14,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41349 2024-11-27T19:14:14,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41349 2024-11-27T19:14:14,059 DEBUG [M:0;eb850308c015:39035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eb850308c015:39035 2024-11-27T19:14:14,062 INFO [master/eb850308c015:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.56.v20240826; built: 2024-08-26T17:15:05.868Z; git: ec6782ff5ead824dabdcf47fa98f90a4aedff401; jvm 17.0.11+9 2024-11-27T19:14:14,065 INFO [master/eb850308c015:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@1fed14f9{HTTP/1.1, (http/1.1)}{0.0.0.0:39251} 2024-11-27T19:14:14,065 INFO [master/eb850308c015:0:becomeActiveMaster {}] server.Server(415): Started @7300ms 2024-11-27T19:14:14,065 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /1/backup-masters/eb850308c015,39035,1732734852191 2024-11-27T19:14:14,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-27T19:14:14,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-27T19:14:14,077 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/eb850308c015,39035,1732734852191 2024-11-27T19:14:14,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:14,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2024-11-27T19:14:14,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:14,101 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/master 2024-11-27T19:14:14,102 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/eb850308c015,39035,1732734852191 from backup master directory 2024-11-27T19:14:14,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/eb850308c015,39035,1732734852191 2024-11-27T19:14:14,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-27T19:14:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2024-11-27T19:14:14,107 WARN [master/eb850308c015:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T19:14:14,107 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eb850308c015,39035,1732734852191 2024-11-27T19:14:14,109 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T19:14:14,111 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T19:14:14,174 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/hbase.id] with ID: f215ccd9-b3b1-442e-b841-1d7c367d5f1f 2024-11-27T19:14:14,174 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.tmp/hbase.id 2024-11-27T19:14:14,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741826_1002 (size=42) 2024-11-27T19:14:14,586 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.tmp/hbase.id]:[hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/hbase.id] 2024-11-27T19:14:14,629 INFO [master/eb850308c015:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T19:14:14,634 INFO [master/eb850308c015:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T19:14:14,656 INFO [master/eb850308c015:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-27T19:14:14,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:14,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:14,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741827_1003 (size=196) 2024-11-27T19:14:15,093 INFO [master/eb850308c015:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:15,095 INFO [master/eb850308c015:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T19:14:15,101 INFO [master/eb850308c015:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-27T19:14:15,104 INFO [master/eb850308c015:0:becomeActiveMaster {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-27T19:14:15,115 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.doInit(RegionGroupingProvider.java:156) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALProvider.init(AbstractWALProvider.java:126) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:263) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T19:14:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741828_1004 (size=1189) 2024-11-27T19:14:15,572 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store 2024-11-27T19:14:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741829_1005 (size=34) 2024-11-27T19:14:15,594 INFO [master/eb850308c015:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T19:14:15,597 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:15,598 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T19:14:15,599 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:14:15,599 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:14:15,600 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T19:14:15,600 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:14:15,601 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:14:15,602 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732734855598Disabling compacts and flushes for region at 1732734855598Disabling writes for close at 1732734855600 (+2 ms)Writing region close event to WAL at 1732734855601 (+1 ms)Closed at 1732734855601 2024-11-27T19:14:15,603 WARN [master/eb850308c015:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/.initializing 2024-11-27T19:14:15,604 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/WALs/eb850308c015,39035,1732734852191 2024-11-27T19:14:15,608 INFO [master/eb850308c015:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T19:14:15,613 INFO [master/eb850308c015:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eb850308c015,39035,1732734852191.regiongroup-0 2024-11-27T19:14:15,628 INFO [master/eb850308c015:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb850308c015%2C39035%2C1732734852191.eb850308c015%2C39035%2C1732734852191.regiongroup-0, suffix=, logDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/WALs/eb850308c015,39035,1732734852191, archiveDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/oldWALs, maxLogs=10 2024-11-27T19:14:15,653 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/WALs/eb850308c015,39035,1732734852191/eb850308c015%2C39035%2C1732734852191.eb850308c015%2C39035%2C1732734852191.regiongroup-0.1732734855632, exclude list is [], retry=0 2024-11-27T19:14:15,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:15,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T19:14:15,711 INFO [master/eb850308c015:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/WALs/eb850308c015,39035,1732734852191/eb850308c015%2C39035%2C1732734852191.eb850308c015%2C39035%2C1732734852191.regiongroup-0.1732734855632 2024-11-27T19:14:15,712 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:15,712 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:15,713 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:15,716 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,717 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,790 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T19:14:15,794 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:15,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:15,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T19:14:15,800 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:15,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:15,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T19:14:15,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:15,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:15,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T19:14:15,807 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:15,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:15,808 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,813 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,814 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,822 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,822 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,827 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T19:14:15,831 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T19:14:15,836 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:15,838 INFO [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10181962400, jitterRate=-0.05173085629940033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T19:14:15,847 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732734855735Initializing all the Stores at 1732734855738 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734855738Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734855740 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734855740Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734855740Cleaning up temporary data from old regions at 1732734855823 (+83 ms)Region opened successfully at 1732734855847 (+24 ms) 2024-11-27T19:14:15,849 INFO [master/eb850308c015:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T19:14:15,887 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a76e11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=eb850308c015/172.17.0.2:0 2024-11-27T19:14:15,930 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T19:14:15,943 INFO [master/eb850308c015:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T19:14:15,943 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T19:14:15,946 INFO [master/eb850308c015:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T19:14:15,948 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T19:14:15,953 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T19:14:15,953 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T19:14:15,980 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T19:14:15,991 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2024-11-27T19:14:15,994 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/balancer already deleted, retry=false 2024-11-27T19:14:15,996 INFO [master/eb850308c015:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T19:14:15,998 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2024-11-27T19:14:15,999 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/normalizer already deleted, retry=false 2024-11-27T19:14:16,001 INFO [master/eb850308c015:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T19:14:16,005 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2024-11-27T19:14:16,006 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/split already deleted, retry=false 2024-11-27T19:14:16,007 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2024-11-27T19:14:16,009 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/switch/merge already deleted, retry=false 2024-11-27T19:14:16,028 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T19:14:16,031 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /1/snapshot-cleanup already deleted, retry=false 2024-11-27T19:14:16,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-27T19:14:16,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2024-11-27T19:14:16,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,056 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eb850308c015,39035,1732734852191, sessionid=0x1015ad983930000, setting cluster-up flag (Was=false) 2024-11-27T19:14:16,066 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2024-11-27T19:14:16,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,082 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2024-11-27T19:14:16,087 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eb850308c015,39035,1732734852191 2024-11-27T19:14:16,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,098 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2024-11-27T19:14:16,100 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eb850308c015,39035,1732734852191 2024-11-27T19:14:16,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,112 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2024-11-27T19:14:16,114 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eb850308c015,39035,1732734852191 2024-11-27T19:14:16,121 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T19:14:16,159 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(746): ClusterId : f215ccd9-b3b1-442e-b841-1d7c367d5f1f 2024-11-27T19:14:16,160 INFO [RS:0;eb850308c015:41349 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2024-11-27T19:14:16,162 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T19:14:16,167 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T19:14:16,167 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2024-11-27T19:14:16,170 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2024-11-27T19:14:16,170 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T19:14:16,173 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T19:14:16,173 DEBUG [RS:0;eb850308c015:41349 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523624ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=eb850308c015/172.17.0.2:0 2024-11-27T19:14:16,187 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eb850308c015:41349 2024-11-27T19:14:16,190 INFO [RS:0;eb850308c015:41349 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T19:14:16,190 INFO [RS:0;eb850308c015:41349 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T19:14:16,190 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T19:14:16,192 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(2659): reportForDuty to master=eb850308c015,39035,1732734852191 with port=41349, startcode=1732734853907 2024-11-27T19:14:16,201 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T19:14:16,206 DEBUG [RS:0;eb850308c015:41349 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T19:14:16,211 INFO [master/eb850308c015:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T19:14:16,217 INFO [master/eb850308c015:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T19:14:16,222 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eb850308c015,39035,1732734852191 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T19:14:16,229 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eb850308c015:0, corePoolSize=5, maxPoolSize=5 2024-11-27T19:14:16,230 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eb850308c015:0, corePoolSize=5, maxPoolSize=5 2024-11-27T19:14:16,230 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eb850308c015:0, corePoolSize=5, maxPoolSize=5 2024-11-27T19:14:16,230 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eb850308c015:0, corePoolSize=5, maxPoolSize=5 2024-11-27T19:14:16,230 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eb850308c015:0, corePoolSize=10, maxPoolSize=10 2024-11-27T19:14:16,230 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,231 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eb850308c015:0, corePoolSize=2, maxPoolSize=2 2024-11-27T19:14:16,231 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,237 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T19:14:16,237 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T19:14:16,239 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732734886239 2024-11-27T19:14:16,241 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T19:14:16,242 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T19:14:16,244 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:16,244 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T19:14:16,248 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T19:14:16,250 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2024-11-27T19:14:16,250 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T19:14:16,250 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T19:14:16,250 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T19:14:16,251 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50473, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T19:14:16,251 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,255 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T19:14:16,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741831_1007 (size=1321) 2024-11-27T19:14:16,257 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T19:14:16,257 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T19:14:16,259 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39035 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T19:14:16,271 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a91062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:16,271 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:16,276 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:16,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:16,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:16,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:16,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5462f7f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:16,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:16,289 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T19:14:16,290 WARN [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T19:14:16,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:16,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:16,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:16,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2543c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:16,297 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2024-11-27T19:14:16,300 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T19:14:16,300 INFO [master/eb850308c015:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T19:14:16,302 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eb850308c015:0:becomeActiveMaster-HFileCleaner.large.0-1732734856302,5,FailOnTimeoutGroup] 2024-11-27T19:14:16,303 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eb850308c015:0:becomeActiveMaster-HFileCleaner.small.0-1732734856302,5,FailOnTimeoutGroup] 2024-11-27T19:14:16,303 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,303 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T19:14:16,304 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,304 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,391 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(2659): reportForDuty to master=eb850308c015,39035,1732734852191 with port=41349, startcode=1732734853907 2024-11-27T19:14:16,393 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39035 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eb850308c015,41349,1732734853907 2024-11-27T19:14:16,395 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39035 {}] master.ServerManager(517): Registering regionserver=eb850308c015,41349,1732734853907 2024-11-27T19:14:16,404 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:16,404 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44785 2024-11-27T19:14:16,404 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=46687 2024-11-27T19:14:16,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-27T19:14:16,414 DEBUG [RS:0;eb850308c015:41349 {}] zookeeper.ZKUtil(111): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rs/eb850308c015,41349,1732734853907 2024-11-27T19:14:16,414 WARN [RS:0;eb850308c015:41349 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T19:14:16,414 INFO [RS:0;eb850308c015:41349 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-27T19:14:16,415 INFO [RS:0;eb850308c015:41349 {}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-27T19:14:16,415 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907 2024-11-27T19:14:16,417 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eb850308c015,41349,1732734853907] 2024-11-27T19:14:16,441 INFO [RS:0;eb850308c015:41349 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T19:14:16,454 INFO [RS:0;eb850308c015:41349 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T19:14:16,460 INFO [RS:0;eb850308c015:41349 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T19:14:16,460 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,461 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T19:14:16,468 INFO [RS:0;eb850308c015:41349 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T19:14:16,469 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,469 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,469 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eb850308c015:0, corePoolSize=2, maxPoolSize=2 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,470 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,471 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eb850308c015:0, corePoolSize=1, maxPoolSize=1 2024-11-27T19:14:16,471 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0, corePoolSize=3, maxPoolSize=3 2024-11-27T19:14:16,471 DEBUG [RS:0;eb850308c015:41349 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eb850308c015:0, corePoolSize=3, maxPoolSize=3 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,472 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,41349,1732734853907-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T19:14:16,490 INFO [RS:0;eb850308c015:41349 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T19:14:16,492 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,41349,1732734853907-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,492 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,492 INFO [RS:0;eb850308c015:41349 {}] regionserver.Replication(171): eb850308c015,41349,1732734853907 started 2024-11-27T19:14:16,509 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:16,509 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1482): Serving as eb850308c015,41349,1732734853907, RpcServer on eb850308c015/172.17.0.2:41349, sessionid=0x1015ad983930001 2024-11-27T19:14:16,510 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T19:14:16,510 DEBUG [RS:0;eb850308c015:41349 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eb850308c015,41349,1732734853907 2024-11-27T19:14:16,510 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb850308c015,41349,1732734853907' 2024-11-27T19:14:16,511 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2024-11-27T19:14:16,512 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2024-11-27T19:14:16,512 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T19:14:16,512 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2024-11-27T19:14:16,512 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb850308c015,41349,1732734853907' 2024-11-27T19:14:16,512 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:14:16,513 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:14:16,514 INFO [RS:0;eb850308c015:41349 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eb850308c015,41349,1732734853907 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb850308c015,41349,1732734853907' 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2024-11-27T19:14:16,514 DEBUG [RS:0;eb850308c015:41349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2024-11-27T19:14:16,515 DEBUG [RS:0;eb850308c015:41349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T19:14:16,515 INFO [RS:0;eb850308c015:41349 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T19:14:16,515 INFO [RS:0;eb850308c015:41349 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T19:14:16,621 INFO [RS:0;eb850308c015:41349 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T19:14:16,621 INFO [RS:0;eb850308c015:41349 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eb850308c015,41349,1732734853907.regiongroup-0 2024-11-27T19:14:16,625 INFO [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0, suffix=, logDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907, archiveDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs, maxLogs=32 2024-11-27T19:14:16,641 DEBUG [RS:0;eb850308c015:41349 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627, exclude list is [], retry=0 2024-11-27T19:14:16,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:16,650 INFO [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 2024-11-27T19:14:16,650 DEBUG [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:16,658 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T19:14:16,659 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741833_1009 (size=32) 2024-11-27T19:14:16,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:16,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T19:14:16,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T19:14:16,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:16,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:16,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T19:14:16,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T19:14:16,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:16,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:16,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T19:14:16,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T19:14:16,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:16,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:16,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T19:14:16,686 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T19:14:16,686 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:16,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:16,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T19:14:16,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740 2024-11-27T19:14:16,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740 2024-11-27T19:14:16,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T19:14:16,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T19:14:16,693 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T19:14:16,695 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T19:14:16,699 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:16,700 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9665269920, jitterRate=-0.099851593375206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T19:14:16,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732734856672Initializing all the Stores at 1732734856674 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734856674Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734856674Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734856674Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734856674Cleaning up temporary data from old regions at 1732734856692 (+18 ms)Region opened successfully at 1732734856703 (+11 ms) 2024-11-27T19:14:16,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T19:14:16,703 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T19:14:16,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T19:14:16,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T19:14:16,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T19:14:16,704 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T19:14:16,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732734856703Disabling compacts and flushes for region at 1732734856703Disabling writes for close at 1732734856703Writing region close event to WAL at 1732734856704 (+1 ms)Closed at 1732734856704 2024-11-27T19:14:16,709 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T19:14:16,709 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T19:14:16,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T19:14:16,725 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T19:14:16,727 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T19:14:16,880 DEBUG [eb850308c015:39035 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T19:14:16,892 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:16,899 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb850308c015,41349,1732734853907, state=OPENING 2024-11-27T19:14:16,904 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T19:14:16,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:14:16,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-27T19:14:16,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-27T19:14:16,908 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T19:14:16,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:17,085 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T19:14:17,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48273, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T19:14:17,100 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T19:14:17,101 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2024-11-27T19:14:17,101 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(102): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2024-11-27T19:14:17,101 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T19:14:17,102 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T19:14:17,106 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb850308c015%2C41349%2C1732734853907.meta, suffix=.meta, logDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907, archiveDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs, maxLogs=32 2024-11-27T19:14:17,123 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta, exclude list is [], retry=0 2024-11-27T19:14:17,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:17,130 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta 2024-11-27T19:14:17,131 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:17,131 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:17,136 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:17,137 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T19:14:17,140 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T19:14:17,140 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T19:14:17,144 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T19:14:17,145 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:17,145 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T19:14:17,145 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T19:14:17,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T19:14:17,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T19:14:17,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:17,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:17,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T19:14:17,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T19:14:17,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:17,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:17,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T19:14:17,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T19:14:17,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:17,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:17,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T19:14:17,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T19:14:17,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:17,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T19:14:17,162 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T19:14:17,164 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740 2024-11-27T19:14:17,167 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740 2024-11-27T19:14:17,170 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T19:14:17,170 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T19:14:17,171 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T19:14:17,174 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T19:14:17,176 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10872062560, jitterRate=0.012539729475975037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T19:14:17,177 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T19:14:17,178 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732734857146Writing region info on filesystem at 1732734857146Initializing all the Stores at 1732734857148 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734857148Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734857148Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734857148Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732734857148Cleaning up temporary data from old regions at 1732734857170 (+22 ms)Running coprocessor post-open hooks at 1732734857177 (+7 ms)Region opened successfully at 1732734857178 (+1 ms) 2024-11-27T19:14:17,186 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732734857076 2024-11-27T19:14:17,197 DEBUG [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T19:14:17,197 INFO [RS_OPEN_META-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T19:14:17,199 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:17,201 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb850308c015,41349,1732734853907, state=OPEN 2024-11-27T19:14:17,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-27T19:14:17,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2024-11-27T19:14:17,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-27T19:14:17,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2024-11-27T19:14:17,203 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eb850308c015,41349,1732734853907 2024-11-27T19:14:17,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T19:14:17,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb850308c015,41349,1732734853907 in 294 msec 2024-11-27T19:14:17,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T19:14:17,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 494 msec 2024-11-27T19:14:17,215 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T19:14:17,215 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T19:14:17,237 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:17,238 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:17,260 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:17,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60769, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:17,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1290 sec 2024-11-27T19:14:17,285 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732734857285, completionTime=-1 2024-11-27T19:14:17,289 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T19:14:17,289 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T19:14:17,326 INFO [master/eb850308c015:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T19:14:17,327 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732734917327 2024-11-27T19:14:17,327 INFO [master/eb850308c015:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732734977327 2024-11-27T19:14:17,327 INFO [master/eb850308c015:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-11-27T19:14:17,329 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,329 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,329 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,331 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eb850308c015:39035, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,331 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,332 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,340 DEBUG [master/eb850308c015:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T19:14:17,365 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.258sec 2024-11-27T19:14:17,367 INFO [master/eb850308c015:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T19:14:17,368 INFO [master/eb850308c015:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T19:14:17,369 INFO [master/eb850308c015:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T19:14:17,370 INFO [master/eb850308c015:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T19:14:17,370 INFO [master/eb850308c015:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T19:14:17,370 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T19:14:17,371 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T19:14:17,379 DEBUG [master/eb850308c015:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T19:14:17,380 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T19:14:17,381 INFO [master/eb850308c015:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb850308c015,39035,1732734852191-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T19:14:17,461 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cc25be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:17,463 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T19:14:17,463 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T19:14:17,466 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:17,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:17,473 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5273da3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:17,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:17,478 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:17,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195abe33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:17,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:17,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:17,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eb850308c015,39035,1732734852191 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2305): Starting mini mapreduce cluster... 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/test.cache.data in system properties and HBase conf 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir in system properties and HBase conf 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T19:14:17,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T19:14:17,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/nfs.dump.dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T19:14:17,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T19:14:17,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741835_1011 (size=592039) 2024-11-27T19:14:17,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741836_1012 (size=1663647) 2024-11-27T19:14:19,623 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:19,731 WARN [Thread-251 {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:19,902 INFO [Thread-251 {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:19,905 INFO [Thread-251 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:19,905 INFO [Thread-251 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:19,905 INFO [Thread-251 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:19,909 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@171eacd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:19,909 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a60e5b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:19,909 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-27T19:14:19,910 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:19,917 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:19,917 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:19,917 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:19,923 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:19,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:19,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:20,033 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2024-11-27T19:14:20,033 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2024-11-27T19:14:20,033 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-27T19:14:20,035 INFO [Thread-251 {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-27T19:14:20,075 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:20,187 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:20,404 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:20,426 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a3f7689{cluster,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-35859-hadoop-yarn-common-3_4_1_jar-_-any-1813859350641850080/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-27T19:14:20,426 INFO [Thread-251 {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@73c48ef9{jobhistory,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-42551-hadoop-yarn-common-3_4_1_jar-_-any-14319688942508341977/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-27T19:14:20,427 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:35859} 2024-11-27T19:14:20,427 INFO [Time-limited test {}] server.Server(415): Started @13661ms 2024-11-27T19:14:20,429 INFO [Thread-251 {}] server.AbstractConnector(333): Started ServerConnector@15a81a3d{HTTP/1.1, (http/1.1)}{localhost:42551} 2024-11-27T19:14:20,429 INFO [Thread-251 {}] server.Server(415): Started @13663ms 2024-11-27T19:14:20,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741837_1013 (size=5) 2024-11-27T19:14:21,276 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-27T19:14:21,281 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:21,305 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-27T19:14:21,305 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:21,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:21,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:21,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:21,313 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:21,314 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@496af81f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:21,314 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b50b82e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:21,371 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-27T19:14:21,371 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-27T19:14:21,371 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-27T19:14:21,371 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-27T19:14:21,378 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,384 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,468 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,478 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10a4b359{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-39357-hadoop-yarn-common-3_4_1_jar-_-any-11622072942633777796/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-27T19:14:21,478 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@37d45da6{HTTP/1.1, (http/1.1)}{localhost:39357} 2024-11-27T19:14:21,478 INFO [Time-limited test {}] server.Server(415): Started @14713ms 2024-11-27T19:14:21,612 WARN [Time-limited test {}] tracker.NMLogAggregationStatusTracker(95): Log Aggregation is disabled.So is the LogAggregationStatusTracker. 2024-11-27T19:14:21,615 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:21,629 WARN [Time-limited test {}] servlet.GuiceFilter(102): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2024-11-27T19:14:21,629 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T19:14:21,633 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T19:14:21,633 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T19:14:21,633 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T19:14:21,634 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T19:14:21,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@db00356{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,AVAILABLE} 2024-11-27T19:14:21,635 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ba88f54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,AVAILABLE} 2024-11-27T19:14:21,689 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(116): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2024-11-27T19:14:21,689 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2024-11-27T19:14:21,689 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(113): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2024-11-27T19:14:21,689 INFO [Time-limited test {}] application.WebApplicationImpl(815): Initiating Jersey application, version 'Jersey: 1.19.4 05/24/2017 03:20 PM' 2024-11-27T19:14:21,697 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,700 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,782 INFO [Time-limited test {}] container.GuiceComponentProviderFactory(168): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2024-11-27T19:14:21,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@223872ea{node,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/java.io.tmpdir/jetty-localhost-41467-hadoop-yarn-common-3_4_1_jar-_-any-3178845272592361939/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-27T19:14:21,788 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53a6a2d1{HTTP/1.1, (http/1.1)}{localhost:41467} 2024-11-27T19:14:21,788 INFO [Time-limited test {}] server.Server(415): Started @15023ms 2024-11-27T19:14:21,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2341): Mini mapreduce cluster started 2024-11-27T19:14:21,815 INFO [Time-limited test {}] backup.TestBackupBase(336): ROOTDIR hdfs://localhost:44785/backupUT 2024-11-27T19:14:21,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T19:14:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is eb850308c015,39035,1732734852191 2024-11-27T19:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@484d50f4 2024-11-27T19:14:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T19:14:21,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T19:14:21,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns1'} 2024-11-27T19:14:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns1 2024-11-27T19:14:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T19:14:21,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns1 in 20 msec 2024-11-27T19:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T19:14:21,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns1 completed 2024-11-27T19:14:21,966 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns2'} 2024-11-27T19:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns2 2024-11-27T19:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-27T19:14:21,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns2 in 12 msec 2024-11-27T19:14:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=5 2024-11-27T19:14:22,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns2 completed 2024-11-27T19:14:22,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns3'} 2024-11-27T19:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=6, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns3 2024-11-27T19:14:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-27T19:14:22,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns3 in 12 msec 2024-11-27T19:14:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=6 2024-11-27T19:14:22,184 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns3 completed 2024-11-27T19:14:22,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'ns4'} 2024-11-27T19:14:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=ns4 2024-11-27T19:14:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T19:14:22,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=ns4 in 12 msec 2024-11-27T19:14:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T19:14:22,294 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: ns4 completed 2024-11-27T19:14:22,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test-1732734861815', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test-1732734861815 2024-11-27T19:14:22,313 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:22,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test-1732734861815" procId is: 8 2024-11-27T19:14:22,316 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-27T19:14:22,318 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:22,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741838_1014 (size=367) 2024-11-27T19:14:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-27T19:14:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-27T19:14:22,653 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-27T19:14:22,718 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T19:14:22,721 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T19:14:22,721 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:22,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:14:22,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2024-11-27T19:14:22,723 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T19:14:22,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T19:14:22,724 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T19:14:22,724 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:22,732 INFO [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44c0c476649e6e06a321127ef8d18631, NAME => 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1732734861815', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741839_1015 (size=53) 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(898): Instantiated test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1722): Closing 44c0c476649e6e06a321127ef8d18631, disabling compactions & flushes 2024-11-27T19:14:22,742 INFO [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1755): Closing region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. after waiting 0 ms 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:22,742 INFO [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1973): Closed test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:22,742 DEBUG [RegionOpenAndInit-test-1732734861815-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44c0c476649e6e06a321127ef8d18631: Waiting for close lock at 1732734862742Disabling compacts and flushes for region at 1732734862742Disabling writes for close at 1732734862742Writing region close event to WAL at 1732734862742Closed at 1732734862742 2024-11-27T19:14:22,745 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:22,750 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732734862745"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734862745"}]},"ts":"1732734862745"} 2024-11-27T19:14:22,755 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:22,756 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:22,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732734861815","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734862756"}]},"ts":"1732734862756"} 2024-11-27T19:14:22,763 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732734861815, state=ENABLING in hbase:meta 2024-11-27T19:14:22,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, ASSIGN}] 2024-11-27T19:14:22,767 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, ASSIGN 2024-11-27T19:14:22,768 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:22,920 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:22,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, ASSIGN because future has completed 2024-11-27T19:14:22,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-27T19:14:23,102 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(132): Open test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:23,102 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T19:14:23,102 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] monitor.StreamSlowMonitor(122): New stream slow monitor eb850308c015,41349,1732734853907.regiongroup-1 2024-11-27T19:14:23,105 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1, suffix=, logDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907, archiveDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs, maxLogs=32 2024-11-27T19:14:23,121 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107, exclude list is [], retry=0 2024-11-27T19:14:23,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:23,127 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107 2024-11-27T19:14:23,127 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:23,127 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c0c476649e6e06a321127ef8d18631, NAME => 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:23,128 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:23,128 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732734861815 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,128 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(898): Instantiated test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:23,128 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7794): checking encryption for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,128 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(7797): checking classloading for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,131 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,133 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName f 2024-11-27T19:14:23,133 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:23,134 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:23,135 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1038): replaying wal for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,136 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,136 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,137 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1048): stopping wal replay for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,137 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1060): Cleaning up temporary data for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,140 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1093): writing seq id for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,143 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:23,144 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1114): Opened 44c0c476649e6e06a321127ef8d18631; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10221946720, jitterRate=-0.04800702631473541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T19:14:23,144 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:23,145 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegion(1006): Region open journal for 44c0c476649e6e06a321127ef8d18631: Running coprocessor pre-open hook at 1732734863128Writing region info on filesystem at 1732734863129 (+1 ms)Initializing all the Stores at 1732734863130 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734863130Cleaning up temporary data from old regions at 1732734863137 (+7 ms)Running coprocessor post-open hooks at 1732734863144 (+7 ms)Region opened successfully at 1732734863145 (+1 ms) 2024-11-27T19:14:23,148 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., pid=10, masterSystemTime=1732734863079 2024-11-27T19:14:23,151 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:23,151 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=10}] handler.AssignRegionHandler(153): Opened test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:23,152 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:23,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:23,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T19:14:23,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 in 232 msec 2024-11-27T19:14:23,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T19:14:23,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, ASSIGN in 396 msec 2024-11-27T19:14:23,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:23,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test-1732734861815","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734863167"}]},"ts":"1732734863167"} 2024-11-27T19:14:23,170 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=test-1732734861815, state=ENABLED in hbase:meta 2024-11-27T19:14:23,171 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=8, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test-1732734861815 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:23,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; CreateTableProcedure table=test-1732734861815 in 865 msec 2024-11-27T19:14:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=8 2024-11-27T19:14:23,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test-1732734861815 completed 2024-11-27T19:14:23,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c38731, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:23,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:23,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:23,457 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e5f1da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:23,460 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:23,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422df53b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:23,464 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:23,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732734861815', row='row0', locateType=CURRENT is [region=test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:23,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41349 {}] regionserver.HRegion(8528): writing data to region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. with WAL disabled. Data may be lost in the event of a crash. 2024-11-27T19:14:23,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns2:test-17327348618151', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns2:test-17327348618151 2024-11-27T19:14:23,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:23,612 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:23,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns2" qualifier: "test-17327348618151" procId is: 11 2024-11-27T19:14:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:23,613 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:23,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741841_1017 (size=364) 2024-11-27T19:14:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:23,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:24,025 INFO [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81ee8faa2e16b928d2ef19ee0c78f7f6, NAME => 'ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17327348618151', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741842_1018 (size=50) 2024-11-27T19:14:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:24,434 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(898): Instantiated ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:24,435 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1722): Closing 81ee8faa2e16b928d2ef19ee0c78f7f6, disabling compactions & flushes 2024-11-27T19:14:24,435 INFO [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1755): Closing region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,435 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,435 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. after waiting 0 ms 2024-11-27T19:14:24,435 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,435 INFO [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1973): Closed ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,435 DEBUG [RegionOpenAndInit-ns2:test-17327348618151-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81ee8faa2e16b928d2ef19ee0c78f7f6: Waiting for close lock at 1732734864435Disabling compacts and flushes for region at 1732734864435Disabling writes for close at 1732734864435Writing region close event to WAL at 1732734864435Closed at 1732734864435 2024-11-27T19:14:24,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:24,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732734864437"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734864437"}]},"ts":"1732734864437"} 2024-11-27T19:14:24,440 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:24,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:24,442 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17327348618151","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734864442"}]},"ts":"1732734864442"} 2024-11-27T19:14:24,444 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17327348618151, state=ENABLING in hbase:meta 2024-11-27T19:14:24,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327348618151, region=81ee8faa2e16b928d2ef19ee0c78f7f6, ASSIGN}] 2024-11-27T19:14:24,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327348618151, region=81ee8faa2e16b928d2ef19ee0c78f7f6, ASSIGN 2024-11-27T19:14:24,447 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns2:test-17327348618151, region=81ee8faa2e16b928d2ef19ee0c78f7f6, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:24,598 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=81ee8faa2e16b928d2ef19ee0c78f7f6, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:24,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns2:test-17327348618151, region=81ee8faa2e16b928d2ef19ee0c78f7f6, ASSIGN because future has completed 2024-11-27T19:14:24,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:24,761 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,762 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 81ee8faa2e16b928d2ef19ee0c78f7f6, NAME => 'ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:24,762 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:24,762 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327348618151 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,762 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:24,762 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,762 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,765 INFO [StoreOpener-81ee8faa2e16b928d2ef19ee0c78f7f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,767 INFO [StoreOpener-81ee8faa2e16b928d2ef19ee0c78f7f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ee8faa2e16b928d2ef19ee0c78f7f6 columnFamilyName f 2024-11-27T19:14:24,767 DEBUG [StoreOpener-81ee8faa2e16b928d2ef19ee0c78f7f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:24,768 INFO [StoreOpener-81ee8faa2e16b928d2ef19ee0c78f7f6-1 {}] regionserver.HStore(327): Store=81ee8faa2e16b928d2ef19ee0c78f7f6/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:24,768 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,769 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,770 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,770 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,770 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,773 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,776 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:24,777 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 81ee8faa2e16b928d2ef19ee0c78f7f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9492625440, jitterRate=-0.11593036353588104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T19:14:24,777 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:24,778 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 81ee8faa2e16b928d2ef19ee0c78f7f6: Running coprocessor pre-open hook at 1732734864763Writing region info on filesystem at 1732734864763Initializing all the Stores at 1732734864764 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734864764Cleaning up temporary data from old regions at 1732734864770 (+6 ms)Running coprocessor post-open hooks at 1732734864777 (+7 ms)Region opened successfully at 1732734864778 (+1 ms) 2024-11-27T19:14:24,780 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6., pid=13, masterSystemTime=1732734864756 2024-11-27T19:14:24,783 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,783 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:24,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=81ee8faa2e16b928d2ef19ee0c78f7f6, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:24,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:24,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=12 2024-11-27T19:14:24,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6, server=eb850308c015,41349,1732734853907 in 187 msec 2024-11-27T19:14:24,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T19:14:24,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns2:test-17327348618151, region=81ee8faa2e16b928d2ef19ee0c78f7f6, ASSIGN in 347 msec 2024-11-27T19:14:24,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:24,797 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns2:test-17327348618151","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734864797"}]},"ts":"1732734864797"} 2024-11-27T19:14:24,800 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns2:test-17327348618151, state=ENABLED in hbase:meta 2024-11-27T19:14:24,801 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=11, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns2:test-17327348618151 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:24,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns2:test-17327348618151 in 1.1930 sec 2024-11-27T19:14:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T19:14:25,764 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns2:test-17327348618151 completed 2024-11-27T19:14:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327348618151', row='row0', locateType=CURRENT is [region=ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:25,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41349 {}] regionserver.HRegion(8528): writing data to region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. with WAL disabled. Data may be lost in the event of a crash. 2024-11-27T19:14:25,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns3:test-17327348618152', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns3:test-17327348618152 2024-11-27T19:14:25,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:25,929 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:25,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns3" qualifier: "test-17327348618152" procId is: 14 2024-11-27T19:14:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:25,930 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741843_1019 (size=390) 2024-11-27T19:14:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:26,342 INFO [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf00efa8e14baf8163e951bfbb0a60c6, NAME => 'ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17327348618152', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741844_1020 (size=50) 2024-11-27T19:14:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(898): Instantiated ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1722): Closing cf00efa8e14baf8163e951bfbb0a60c6, disabling compactions & flushes 2024-11-27T19:14:26,751 INFO [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1755): Closing region ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. after waiting 0 ms 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:26,751 INFO [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1973): Closed ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:26,751 DEBUG [RegionOpenAndInit-ns3:test-17327348618152-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf00efa8e14baf8163e951bfbb0a60c6: Waiting for close lock at 1732734866751Disabling compacts and flushes for region at 1732734866751Disabling writes for close at 1732734866751Writing region close event to WAL at 1732734866751Closed at 1732734866751 2024-11-27T19:14:26,753 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:26,753 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732734866753"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734866753"}]},"ts":"1732734866753"} 2024-11-27T19:14:26,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:26,757 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:26,758 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17327348618152","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734866757"}]},"ts":"1732734866757"} 2024-11-27T19:14:26,760 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17327348618152, state=ENABLING in hbase:meta 2024-11-27T19:14:26,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327348618152, region=cf00efa8e14baf8163e951bfbb0a60c6, ASSIGN}] 2024-11-27T19:14:26,762 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327348618152, region=cf00efa8e14baf8163e951bfbb0a60c6, ASSIGN 2024-11-27T19:14:26,763 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns3:test-17327348618152, region=cf00efa8e14baf8163e951bfbb0a60c6, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:26,914 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cf00efa8e14baf8163e951bfbb0a60c6, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:26,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns3:test-17327348618152, region=cf00efa8e14baf8163e951bfbb0a60c6, ASSIGN because future has completed 2024-11-27T19:14:26,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:27,076 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:27,076 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => cf00efa8e14baf8163e951bfbb0a60c6, NAME => 'ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:27,077 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:27,077 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327348618152 cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,077 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:27,077 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,078 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,080 INFO [StoreOpener-cf00efa8e14baf8163e951bfbb0a60c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,081 INFO [StoreOpener-cf00efa8e14baf8163e951bfbb0a60c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf00efa8e14baf8163e951bfbb0a60c6 columnFamilyName f 2024-11-27T19:14:27,082 DEBUG [StoreOpener-cf00efa8e14baf8163e951bfbb0a60c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:27,082 INFO [StoreOpener-cf00efa8e14baf8163e951bfbb0a60c6-1 {}] regionserver.HStore(327): Store=cf00efa8e14baf8163e951bfbb0a60c6/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:27,082 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,083 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns3/test-17327348618152/cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,084 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns3/test-17327348618152/cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,084 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,085 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,087 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,090 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns3/test-17327348618152/cf00efa8e14baf8163e951bfbb0a60c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:27,090 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened cf00efa8e14baf8163e951bfbb0a60c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10647435520, jitterRate=-0.008380293846130371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T19:14:27,091 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:14:27,091 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for cf00efa8e14baf8163e951bfbb0a60c6: Running coprocessor pre-open hook at 1732734867078Writing region info on filesystem at 1732734867078Initializing all the Stores at 1732734867079 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734867079Cleaning up temporary data from old regions at 1732734867085 (+6 ms)Running coprocessor post-open hooks at 1732734867091 (+6 ms)Region opened successfully at 1732734867091 2024-11-27T19:14:27,093 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6., pid=16, masterSystemTime=1732734867071 2024-11-27T19:14:27,095 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:27,095 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:14:27,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=cf00efa8e14baf8163e951bfbb0a60c6, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:27,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:27,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T19:14:27,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6, server=eb850308c015,41349,1732734853907 in 182 msec 2024-11-27T19:14:27,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T19:14:27,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns3:test-17327348618152, region=cf00efa8e14baf8163e951bfbb0a60c6, ASSIGN in 342 msec 2024-11-27T19:14:27,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:27,106 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns3:test-17327348618152","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734867106"}]},"ts":"1732734867106"} 2024-11-27T19:14:27,108 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=ns3:test-17327348618152, state=ENABLED in hbase:meta 2024-11-27T19:14:27,109 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=14, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns3:test-17327348618152 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:27,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns3:test-17327348618152 in 1.1840 sec 2024-11-27T19:14:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=14 2024-11-27T19:14:28,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns3:test-17327348618152 completed 2024-11-27T19:14:28,075 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns3:test-17327348618152 get assigned. Timeout = 60000ms 2024-11-27T19:14:28,076 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T19:14:28,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns3:test-17327348618152 assigned to meta. Checking AM states. 2024-11-27T19:14:28,082 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T19:14:28,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns3:test-17327348618152 assigned. 2024-11-27T19:14:28,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'ns4:test-17327348618153', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=ns4:test-17327348618153 2024-11-27T19:14:28,087 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:28,087 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:28,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "ns4" qualifier: "test-17327348618153" procId is: 17 2024-11-27T19:14:28,088 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:28,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741845_1021 (size=390) 2024-11-27T19:14:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:28,499 INFO [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f9ef0e14ba9a7626cfb7c31375818ae5, NAME => 'ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17327348618153', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:28,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741846_1022 (size=50) 2024-11-27T19:14:28,631 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-27T19:14:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:28,721 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17327348618151' 2024-11-27T19:14:28,721 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17327348618152' 2024-11-27T19:14:28,722 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1732734861815' 2024-11-27T19:14:28,908 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(898): Instantiated ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:28,909 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1722): Closing f9ef0e14ba9a7626cfb7c31375818ae5, disabling compactions & flushes 2024-11-27T19:14:28,909 INFO [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1755): Closing region ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:28,909 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:28,909 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. after waiting 0 ms 2024-11-27T19:14:28,909 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:28,909 INFO [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1973): Closed ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:28,909 DEBUG [RegionOpenAndInit-ns4:test-17327348618153-pool-0 {}] regionserver.HRegion(1676): Region close journal for f9ef0e14ba9a7626cfb7c31375818ae5: Waiting for close lock at 1732734868909Disabling compacts and flushes for region at 1732734868909Disabling writes for close at 1732734868909Writing region close event to WAL at 1732734868909Closed at 1732734868909 2024-11-27T19:14:28,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:28,911 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1732734868911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734868911"}]},"ts":"1732734868911"} 2024-11-27T19:14:28,914 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:28,915 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:28,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17327348618153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734868915"}]},"ts":"1732734868915"} 2024-11-27T19:14:28,921 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17327348618153, state=ENABLING in hbase:meta 2024-11-27T19:14:28,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327348618153, region=f9ef0e14ba9a7626cfb7c31375818ae5, ASSIGN}] 2024-11-27T19:14:28,923 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327348618153, region=f9ef0e14ba9a7626cfb7c31375818ae5, ASSIGN 2024-11-27T19:14:28,924 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=ns4:test-17327348618153, region=f9ef0e14ba9a7626cfb7c31375818ae5, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:29,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f9ef0e14ba9a7626cfb7c31375818ae5, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:29,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=ns4:test-17327348618153, region=f9ef0e14ba9a7626cfb7c31375818ae5, ASSIGN because future has completed 2024-11-27T19:14:29,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:29,235 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:29,236 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => f9ef0e14ba9a7626cfb7c31375818ae5, NAME => 'ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:29,236 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:29,236 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17327348618153 f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,236 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:29,236 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,237 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,238 INFO [StoreOpener-f9ef0e14ba9a7626cfb7c31375818ae5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,241 INFO [StoreOpener-f9ef0e14ba9a7626cfb7c31375818ae5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9ef0e14ba9a7626cfb7c31375818ae5 columnFamilyName f 2024-11-27T19:14:29,241 DEBUG [StoreOpener-f9ef0e14ba9a7626cfb7c31375818ae5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:29,242 INFO [StoreOpener-f9ef0e14ba9a7626cfb7c31375818ae5-1 {}] regionserver.HStore(327): Store=f9ef0e14ba9a7626cfb7c31375818ae5/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:29,242 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,243 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns4/test-17327348618153/f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,244 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns4/test-17327348618153/f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,244 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,244 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,246 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,249 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns4/test-17327348618153/f9ef0e14ba9a7626cfb7c31375818ae5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:29,250 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened f9ef0e14ba9a7626cfb7c31375818ae5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9865702240, jitterRate=-0.08118487894535065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T19:14:29,250 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:14:29,251 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for f9ef0e14ba9a7626cfb7c31375818ae5: Running coprocessor pre-open hook at 1732734869237Writing region info on filesystem at 1732734869237Initializing all the Stores at 1732734869238 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734869238Cleaning up temporary data from old regions at 1732734869244 (+6 ms)Running coprocessor post-open hooks at 1732734869250 (+6 ms)Region opened successfully at 1732734869251 (+1 ms) 2024-11-27T19:14:29,252 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5., pid=19, masterSystemTime=1732734869231 2024-11-27T19:14:29,254 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:29,255 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:14:29,255 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f9ef0e14ba9a7626cfb7c31375818ae5, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:29,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:29,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-27T19:14:29,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5, server=eb850308c015,41349,1732734853907 in 180 msec 2024-11-27T19:14:29,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T19:14:29,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=ns4:test-17327348618153, region=f9ef0e14ba9a7626cfb7c31375818ae5, ASSIGN in 340 msec 2024-11-27T19:14:29,265 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:29,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"ns4:test-17327348618153","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734869266"}]},"ts":"1732734869266"} 2024-11-27T19:14:29,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=ns4:test-17327348618153, state=ENABLED in hbase:meta 2024-11-27T19:14:29,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=17, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=ns4:test-17327348618153 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:29,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; CreateTableProcedure table=ns4:test-17327348618153 in 1.1850 sec 2024-11-27T19:14:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T19:14:30,234 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: ns4:test-17327348618153 completed 2024-11-27T19:14:30,235 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table ns4:test-17327348618153 get assigned. Timeout = 60000ms 2024-11-27T19:14:30,235 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T19:14:30,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table ns4:test-17327348618153 assigned to meta. Checking AM states. 2024-11-27T19:14:30,240 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T19:14:30,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table ns4:test-17327348618153 assigned. 2024-11-27T19:14:30,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:14:30,241 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestBackupBase.createTables(TestBackupBase.java:481) at org.apache.hadoop.hbase.backup.TestBackupBase.setUpHelper(TestBackupBase.java:343) at org.apache.hadoop.hbase.backup.TestBackupBase.setUp(TestBackupBase.java:357) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:14:30,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:30,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:30,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:14:30,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=611, OpenFileDescriptor=633, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=15, AvailableMemoryMB=5721 2024-11-27T19:14:30,283 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=611 is superior to 500 2024-11-27T19:14:30,285 INFO [Time-limited test {}] backup.TestIncrementalBackup(91): create full backup image for all tables 2024-11-27T19:14:30,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732734861815 from 'test-1732734861815', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732734861815', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=20, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732734861815 2024-11-27T19:14:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741847_1023 (size=908) 2024-11-27T19:14:30,717 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000001.367 2024-11-27T19:14:30,717 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000002.908 2024-11-27T19:14:30,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732734861815}] 2024-11-27T19:14:30,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE}] 2024-11-27T19:14:30,730 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE 2024-11-27T19:14:30,732 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=CLOSING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:30,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE because future has completed 2024-11-27T19:14:30,735 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T19:14:30,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:30,892 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(122): Close 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:30,893 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T19:14:30,893 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1722): Closing 44c0c476649e6e06a321127ef8d18631, disabling compactions & flushes 2024-11-27T19:14:30,893 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1755): Closing region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:30,893 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:30,893 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1843): Acquired close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. after waiting 0 ms 2024-11-27T19:14:30,893 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1853): Updates disabled for region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:30,897 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(2902): Flushing 44c0c476649e6e06a321127ef8d18631 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-27T19:14:30,965 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/ee663f7a5913448da72880447b51fed4 is 37, key is row10/f:q1/1732734863493/Put/seqid=0 2024-11-27T19:14:30,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741848_1024 (size=8413) 2024-11-27T19:14:30,977 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:14:31,038 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/ee663f7a5913448da72880447b51fed4 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:14:31,049 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4, entries=99, sequenceid=103, filesize=8.2 K 2024-11-27T19:14:31,058 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 158ms, sequenceid=103, compaction requested=false 2024-11-27T19:14:31,066 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2024-11-27T19:14:31,070 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:14:31,070 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1973): Closed test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:31,070 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegion(1676): Region close journal for 44c0c476649e6e06a321127ef8d18631: Waiting for close lock at 1732734870893Running coprocessor pre-close hooks at 1732734870893Disabling compacts and flushes for region at 1732734870893Disabling writes for close at 1732734870893Obtaining lock to block concurrent updates at 1732734870897 (+4 ms)Preparing flush snapshotting stores in 44c0c476649e6e06a321127ef8d18631 at 1732734870897Finished memstore snapshotting test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., syncing WAL and waiting on mvcc, flushsize=dataSize=3247, getHeapSize=11248, getOffHeapSize=0, getCellsCount=99 at 1732734870905 (+8 ms)Flushing stores of test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. at 1732734870907 (+2 ms)Flushing 44c0c476649e6e06a321127ef8d18631/f: creating writer at 1732734870908 (+1 ms)Flushing 44c0c476649e6e06a321127ef8d18631/f: appending metadata at 1732734870954 (+46 ms)Flushing 44c0c476649e6e06a321127ef8d18631/f: closing flushed file at 1732734870957 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d9e6cd: reopening flushed file at 1732734871037 (+80 ms)Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 158ms, sequenceid=103, compaction requested=false at 1732734871058 (+21 ms)Writing region close event to WAL at 1732734871060 (+2 ms)Running coprocessor post-close hooks at 1732734871067 (+7 ms)Closed at 1732734871070 (+3 ms) 2024-11-27T19:14:31,070 WARN [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] regionserver.HRegionServer(3299): Not adding moved region record: 44c0c476649e6e06a321127ef8d18631 to self. 2024-11-27T19:14:31,074 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=23}] handler.UnassignRegionHandler(157): Closed 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,074 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=CLOSED 2024-11-27T19:14:31,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:31,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-27T19:14:31,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 in 343 msec 2024-11-27T19:14:31,082 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE; state=CLOSED, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=true 2024-11-27T19:14:31,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:31,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE because future has completed 2024-11-27T19:14:31,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:31,393 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:31,393 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c0c476649e6e06a321127ef8d18631, NAME => 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:31,394 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:31,394 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732734861815 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,394 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:31,394 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,394 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,396 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,397 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName f 2024-11-27T19:14:31,397 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:31,406 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:14:31,406 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:31,406 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,408 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName f3 2024-11-27T19:14:31,408 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:31,409 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:31,410 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,416 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName mob 2024-11-27T19:14:31,420 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:31,421 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:31,421 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,422 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,424 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,425 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,425 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,426 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732734861815 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T19:14:31,428 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,429 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 44c0c476649e6e06a321127ef8d18631; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10582944000, jitterRate=-0.014386534690856934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T19:14:31,429 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:31,430 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 44c0c476649e6e06a321127ef8d18631: Running coprocessor pre-open hook at 1732734871394Writing region info on filesystem at 1732734871394Initializing all the Stores at 1732734871395 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734871395Instantiating store for column family {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734871396 (+1 ms)Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734871396Cleaning up temporary data from old regions at 1732734871425 (+29 ms)Running coprocessor post-open hooks at 1732734871429 (+4 ms)Region opened successfully at 1732734871429 2024-11-27T19:14:31,431 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., pid=24, masterSystemTime=1732734871388 2024-11-27T19:14:31,434 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:31,434 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:31,434 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPEN, openSeqNum=107, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:31,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:31,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=22 2024-11-27T19:14:31,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 in 202 msec 2024-11-27T19:14:31,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T19:14:31,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE in 711 msec 2024-11-27T19:14:31,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-27T19:14:31,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732734861815 in 724 msec 2024-11-27T19:14:31,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732734861815 in 1.1570 sec 2024-11-27T19:14:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=20 2024-11-27T19:14:31,454 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732734861815 completed 2024-11-27T19:14:31,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220f37bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:31,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:31,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:31,457 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:31,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:31,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:31,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c2bc181, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:31,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:31,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:31,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:31,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:31,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79102cbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:31,461 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:31,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:31,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:31,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:31,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732734861815', row='row-test-1732734861815-3-0', locateType=CURRENT is [region=test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., hostname=eb850308c015,41349,1732734853907, seqNum=107] 2024-11-27T19:14:31,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T19:14:31,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is eb850308c015,39035,1732734852191 2024-11-27T19:14:31,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32dd323f 2024-11-27T19:14:31,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T19:14:31,516 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T19:14:31,527 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$18(3529): Client=jenkins//172.17.0.2 creating {NAME => 'backup'} 2024-11-27T19:14:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE, hasLock=false; CreateNamespaceProcedure, namespace=backup 2024-11-27T19:14:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T19:14:31,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateNamespaceProcedure, namespace=backup in 10 msec 2024-11-27T19:14:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T19:14:31,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$NamespaceProcedureBiConsumer(2745): Operation: CREATE_NAMESPACE, Namespace: backup completed 2024-11-27T19:14:31,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system 2024-11-27T19:14:31,673 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:31,674 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:31,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2024-11-27T19:14:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:31,675 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:31,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741849_1025 (size=635) 2024-11-27T19:14:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:32,086 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:32,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741850_1026 (size=40) 2024-11-27T19:14:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1722): Closing 4ed13a3963bd22f54d2cd06886c65e56, disabling compactions & flushes 2024-11-27T19:14:32,495 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. after waiting 0 ms 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,495 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1973): Closed backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,495 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4ed13a3963bd22f54d2cd06886c65e56: Waiting for close lock at 1732734872495Disabling compacts and flushes for region at 1732734872495Disabling writes for close at 1732734872495Writing region close event to WAL at 1732734872495Closed at 1732734872495 2024-11-27T19:14:32,497 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:32,498 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732734872497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734872497"}]},"ts":"1732734872497"} 2024-11-27T19:14:32,500 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:32,501 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:32,502 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734872501"}]},"ts":"1732734872501"} 2024-11-27T19:14:32,504 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-27T19:14:32,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN}] 2024-11-27T19:14:32,505 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN 2024-11-27T19:14:32,506 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:32,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17327348618152 2024-11-27T19:14:32,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17327348618152 Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:32,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17327348618151 2024-11-27T19:14:32,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17327348618151 Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:32,639 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17327348618153 2024-11-27T19:14:32,639 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17327348618153 Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:32,639 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1732734861815 2024-11-27T19:14:32,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1732734861815 Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:32,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:32,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN because future has completed 2024-11-27T19:14:32,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:32,820 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,820 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:32,820 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:32,821 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,821 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:32,821 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,821 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,823 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,825 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed13a3963bd22f54d2cd06886c65e56 columnFamilyName meta 2024-11-27T19:14:32,826 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:32,826 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(327): Store=4ed13a3963bd22f54d2cd06886c65e56/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:32,826 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,829 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed13a3963bd22f54d2cd06886c65e56 columnFamilyName session 2024-11-27T19:14:32,829 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:32,829 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(327): Store=4ed13a3963bd22f54d2cd06886c65e56/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:32,830 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,831 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,831 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,833 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,833 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,835 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T19:14:32,837 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,840 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:32,841 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 4ed13a3963bd22f54d2cd06886c65e56; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11017450880, jitterRate=0.026080071926116943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T19:14:32,841 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:32,841 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 4ed13a3963bd22f54d2cd06886c65e56: Running coprocessor pre-open hook at 1732734872821Writing region info on filesystem at 1732734872821Initializing all the Stores at 1732734872823 (+2 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734872823Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734872823Cleaning up temporary data from old regions at 1732734872833 (+10 ms)Running coprocessor post-open hooks at 1732734872841 (+8 ms)Region opened successfully at 1732734872841 2024-11-27T19:14:32,842 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., pid=28, masterSystemTime=1732734872814 2024-11-27T19:14:32,845 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,845 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:32,846 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:32,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:32,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T19:14:32,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 in 188 msec 2024-11-27T19:14:32,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T19:14:32,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN in 348 msec 2024-11-27T19:14:32,856 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:32,857 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734872856"}]},"ts":"1732734872856"} 2024-11-27T19:14:32,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-27T19:14:32,860 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:32,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system in 1.1890 sec 2024-11-27T19:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-27T19:14:33,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system completed 2024-11-27T19:14:33,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=backup:system_bulk 2024-11-27T19:14:33,856 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T19:14:33,856 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:33,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2024-11-27T19:14:33,857 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T19:14:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741851_1027 (size=640) 2024-11-27T19:14:33,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:34,267 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 05e9d065d2547a02238ecbaeb2f3acbc, NAME => 'backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:34,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741852_1028 (size=45) 2024-11-27T19:14:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:34,676 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:34,677 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1722): Closing 05e9d065d2547a02238ecbaeb2f3acbc, disabling compactions & flushes 2024-11-27T19:14:34,677 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:34,677 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:34,677 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. after waiting 0 ms 2024-11-27T19:14:34,677 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:34,677 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:34,677 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1676): Region close journal for 05e9d065d2547a02238ecbaeb2f3acbc: Waiting for close lock at 1732734874677Disabling compacts and flushes for region at 1732734874677Disabling writes for close at 1732734874677Writing region close event to WAL at 1732734874677Closed at 1732734874677 2024-11-27T19:14:34,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T19:14:34,679 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732734874679"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734874679"}]},"ts":"1732734874679"} 2024-11-27T19:14:34,682 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:14:34,683 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T19:14:34,684 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734874683"}]},"ts":"1732734874683"} 2024-11-27T19:14:34,686 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2024-11-27T19:14:34,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=05e9d065d2547a02238ecbaeb2f3acbc, ASSIGN}] 2024-11-27T19:14:34,688 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=05e9d065d2547a02238ecbaeb2f3acbc, ASSIGN 2024-11-27T19:14:34,689 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=05e9d065d2547a02238ecbaeb2f3acbc, ASSIGN; state=OFFLINE, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=false 2024-11-27T19:14:34,738 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-27T19:14:34,766 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2024-11-27T19:14:34,766 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17327348618153' 2024-11-27T19:14:34,839 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=05e9d065d2547a02238ecbaeb2f3acbc, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:34,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system_bulk, region=05e9d065d2547a02238ecbaeb2f3acbc, ASSIGN because future has completed 2024-11-27T19:14:34,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:14:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:35,000 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:35,000 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 05e9d065d2547a02238ecbaeb2f3acbc, NAME => 'backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:14:35,001 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:14:35,001 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,001 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:14:35,001 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,001 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,003 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,004 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05e9d065d2547a02238ecbaeb2f3acbc columnFamilyName meta 2024-11-27T19:14:35,005 DEBUG [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:35,005 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] regionserver.HStore(327): Store=05e9d065d2547a02238ecbaeb2f3acbc/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:35,005 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,007 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05e9d065d2547a02238ecbaeb2f3acbc columnFamilyName session 2024-11-27T19:14:35,007 DEBUG [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:35,007 INFO [StoreOpener-05e9d065d2547a02238ecbaeb2f3acbc-1 {}] regionserver.HStore(327): Store=05e9d065d2547a02238ecbaeb2f3acbc/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:14:35,008 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,008 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system_bulk/05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,009 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system_bulk/05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,010 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,010 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,011 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T19:14:35,012 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,015 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system_bulk/05e9d065d2547a02238ecbaeb2f3acbc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T19:14:35,015 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 05e9d065d2547a02238ecbaeb2f3acbc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10563419840, jitterRate=-0.016204863786697388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T19:14:35,015 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:14:35,016 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 05e9d065d2547a02238ecbaeb2f3acbc: Running coprocessor pre-open hook at 1732734875001Writing region info on filesystem at 1732734875001Initializing all the Stores at 1732734875002 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734875002Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734875003 (+1 ms)Cleaning up temporary data from old regions at 1732734875010 (+7 ms)Running coprocessor post-open hooks at 1732734875016 (+6 ms)Region opened successfully at 1732734875016 2024-11-27T19:14:35,017 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc., pid=31, masterSystemTime=1732734874995 2024-11-27T19:14:35,020 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:35,020 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:14:35,021 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=05e9d065d2547a02238ecbaeb2f3acbc, regionState=OPEN, openSeqNum=2, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:14:35,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:14:35,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T19:14:35,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc, server=eb850308c015,41349,1732734853907 in 182 msec 2024-11-27T19:14:35,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T19:14:35,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system_bulk, region=05e9d065d2547a02238ecbaeb2f3acbc, ASSIGN in 342 msec 2024-11-27T19:14:35,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T19:14:35,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734875032"}]},"ts":"1732734875032"} 2024-11-27T19:14:35,034 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2024-11-27T19:14:35,035 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T19:14:35,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; CreateTableProcedure table=backup:system_bulk in 1.1820 sec 2024-11-27T19:14:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T19:14:36,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: backup:system_bulk completed 2024-11-27T19:14:36,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:36,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:36,033 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-27T19:14:36,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='activesession:', locateType=CURRENT is [region=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41349 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2024-11-27T19:14:36,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734876065 (current time:1732734876065). 2024-11-27T19:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:14:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-27T19:14:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29307c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:36,068 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:36,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:36,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:36,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613e6080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:36,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:36,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:36,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:36,071 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:36,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1297636c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:36,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:36,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:36,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:36,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:14:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:36,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:14:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:14:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:14:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-27T19:14:36,090 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:14:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:36,094 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:14:36,109 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:14:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741853_1029 (size=52) 2024-11-27T19:14:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:36,518 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:14:36,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56}] 2024-11-27T19:14:36,525 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:36,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-27T19:14:36,682 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:36,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=45 B heapSize=632 B 2024-11-27T19:14:36,702 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/df512fd414b94e848ba4beffec983c91 is 49, key is activesession:/session:c/1732734876051/Put/seqid=0 2024-11-27T19:14:36,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741854_1030 (size=5049) 2024-11-27T19:14:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:37,109 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/df512fd414b94e848ba4beffec983c91 2024-11-27T19:14:37,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/df512fd414b94e848ba4beffec983c91 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 2024-11-27T19:14:37,128 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, entries=1, sequenceid=5, filesize=4.9 K 2024-11-27T19:14:37,129 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 446ms, sequenceid=5, compaction requested=false 2024-11-27T19:14:37,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:14:37,129 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. for snapshot_backup_system completed. 2024-11-27T19:14:37,131 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.' region-info for snapshot=snapshot_backup_system 2024-11-27T19:14:37,135 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:14:37,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:14:37,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91] hfiles 2024-11-27T19:14:37,141 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 for snapshot=snapshot_backup_system 2024-11-27T19:14:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741855_1031 (size=95) 2024-11-27T19:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:37,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:37,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-27T19:14:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-27T19:14:37,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:37,559 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:37,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T19:14:37,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 in 1.0400 sec 2024-11-27T19:14:37,563 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:14:37,566 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:14:37,567 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:14:37,567 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-27T19:14:37,570 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-27T19:14:37,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741856_1032 (size=731) 2024-11-27T19:14:37,989 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:14:38,000 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:14:38,000 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_backup_system 2024-11-27T19:14:38,002 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:14:38,002 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2024-11-27T19:14:38,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9170 sec 2024-11-27T19:14:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-27T19:14:38,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-27T19:14:38,234 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732734871496 started at 1732734878233. 2024-11-27T19:14:38,248 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732734871496 has been started. 2024-11-27T19:14:38,253 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-27T19:14:38,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-27T19:14:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-27T19:14:38,261 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-27T19:14:38,261 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-27T19:14:38,261 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-27T19:14:38,261 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-27T19:14:38,262 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,262 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:38,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:14:38,265 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,266 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:14:38,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:14:38,266 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,266 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-27T19:14:38,266 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:38,267 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-27T19:14:38,268 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:38,268 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-27T19:14:38,269 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-27T19:14:38,269 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-27T19:14:38,270 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-27T19:14:38,270 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-27T19:14:38,270 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-27T19:14:38,270 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-27T19:14:38,270 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure (rolllog) in zk 2024-11-27T19:14:38,272 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:38,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:38,273 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,273 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-27T19:14:38,273 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:38,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:38,274 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:38,275 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:38,276 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-27T19:14:38,276 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@73eddf19[Count = 0] remaining members to acquire global barrier 2024-11-27T19:14:38,276 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-27T19:14:38,276 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,278 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,278 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,278 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,278 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-27T19:14:38,278 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-27T19:14:38,279 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-27T19:14:38,279 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: eb850308c015,41349,1732734853907 2024-11-27T19:14:38,285 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732734856627 highest: 1732734863107 on eb850308c015,41349,1732734853907 2024-11-27T19:14:38,285 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0:(num 1732734856627) roll requested 2024-11-27T19:14:38,299 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286, exclude list is [], retry=0 2024-11-27T19:14:38,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:38,305 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 with entries=3, filesize=1.00 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 2024-11-27T19:14:38,305 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:38,305 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 is not closed yet, will try archiving it next time 2024-11-27T19:14:38,306 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1:(num 1732734863107) roll requested 2024-11-27T19:14:38,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741832_1008 (size=1034) 2024-11-27T19:14:38,308 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 not finished, retry = 0 2024-11-27T19:14:38,319 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306, exclude list is [], retry=0 2024-11-27T19:14:38,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:38,325 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107 with entries=24, filesize=4.64 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 2024-11-27T19:14:38,325 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:38,325 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107 is not closed yet, will try archiving it next time 2024-11-27T19:14:38,325 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.meta:.meta(num 1732734857108) roll requested 2024-11-27T19:14:38,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741840_1016 (size=4756) 2024-11-27T19:14:38,339 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta, exclude list is [], retry=0 2024-11-27T19:14:38,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:38,344 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta with entries=42, filesize=11.76 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta 2024-11-27T19:14:38,344 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:38,344 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta is not closed yet, will try archiving it next time 2024-11-27T19:14:38,346 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-27T19:14:38,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741834_1010 (size=12055) 2024-11-27T19:14:38,346 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732734878286 on eb850308c015,41349,1732734853907 2024-11-27T19:14:38,346 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T19:14:38,347 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] client.AsyncConnectionImpl(321): The fetched master address is eb850308c015,39035,1732734852191 2024-11-27T19:14:38,347 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42ce6684 2024-11-27T19:14:38,347 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T19:14:38,348 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50979, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2024-11-27T19:14:38,349 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:38,349 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:38,350 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:38,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-11-27T19:14:38,359 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:38,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:38,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='rslogts:hdfs://localhost:44785/backupUT\x00', locateType=CURRENT is [region=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:38,373 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-27T19:14:38,373 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'eb850308c015,41349,1732734853907' in zk 2024-11-27T19:14:38,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,375 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-27T19:14:38,375 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:14:38,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:38,375 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-27T19:14:38,375 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:38,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:38,376 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:38,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,377 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:38,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:38,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:38,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'eb850308c015,41349,1732734853907': 2024-11-27T19:14:38,380 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-27T19:14:38,380 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-27T19:14:38,380 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-27T19:14:38,380 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-27T19:14:38,380 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-27T19:14:38,380 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'eb850308c015,41349,1732734853907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-27T19:14:38,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:14:38,384 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:38,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:38,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:38,385 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:14:38,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:14:38,385 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:38,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:38,387 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:38,388 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:38,389 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:38,395 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,395 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:38,395 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:14:38,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:38,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-27T19:14:38,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-27T19:14:38,396 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:14:38,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:14:38,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-27T19:14:38,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } 2024-11-27T19:14:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734878410 (current time:1732734878410). 2024-11-27T19:14:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:14:38,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734878409_default_test-1732734861815 VERSION not specified, setting to 2 2024-11-27T19:14:38,410 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734856627 2024-11-27T19:14:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298e8612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:38,412 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:38,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:38,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:38,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b546d5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:38,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:38,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:38,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:38,414 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:38,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bd0363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:38,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:38,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:38,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:38,418 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:14:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:38,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:14:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:14:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } 2024-11-27T19:14:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-27T19:14:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:38,421 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:14:38,422 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:14:38,425 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:14:38,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741860_1036 (size=84) 2024-11-27T19:14:38,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:38,832 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:14:38,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631}] 2024-11-27T19:14:38,834 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:38,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-27T19:14:38,986 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:38,987 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 44c0c476649e6e06a321127ef8d18631 3/3 column families, dataSize=654 B heapSize=2.25 KB 2024-11-27T19:14:39,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f3/342aba8c72f746219de51bde2e2ec310 is 58, key is row-test-1732734861815-3-0/f3:q1/1732734871467/Put/seqid=0 2024-11-27T19:14:39,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741861_1037 (size=5387) 2024-11-27T19:14:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:39,411 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f3/342aba8c72f746219de51bde2e2ec310 2024-11-27T19:14:39,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741862_1038 (size=4469) 2024-11-27T19:14:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:39,853 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/b947d32e22564efb834122adebb606cf is 59, key is row-test-1732734861815-3-0/mob:q1/1732734871481/Put/seqid=0 2024-11-27T19:14:39,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741863_1039 (size=5484) 2024-11-27T19:14:40,261 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=330, hasBloomFilter=true, into tmp file hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/b947d32e22564efb834122adebb606cf 2024-11-27T19:14:40,270 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f3/342aba8c72f746219de51bde2e2ec310 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 2024-11-27T19:14:40,278 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310, entries=6, sequenceid=121, filesize=5.3 K 2024-11-27T19:14:40,279 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/b947d32e22564efb834122adebb606cf as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf 2024-11-27T19:14:40,288 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf, entries=6, sequenceid=121, filesize=5.4 K 2024-11-27T19:14:40,289 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~654 B/654, heapSize ~1.97 KB/2016, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 1303ms, sequenceid=121, compaction requested=false 2024-11-27T19:14:40,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 44c0c476649e6e06a321127ef8d18631: 2024-11-27T19:14:40,289 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. for snapshot_1732734878409_default_test-1732734861815 completed. 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.' region-info for snapshot=snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4] hfiles 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 for snapshot=snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310] hfiles 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 for snapshot=snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf] hfiles 2024-11-27T19:14:40,290 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf for snapshot=snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:40,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741864_1040 (size=185) 2024-11-27T19:14:40,503 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2024-11-27T19:14:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:40,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:40,697 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-27T19:14:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-27T19:14:40,698 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734878409_default_test-1732734861815 on region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:40,698 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:40,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-27T19:14:40,702 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:14:40,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 in 1.8660 sec 2024-11-27T19:14:40,703 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:14:40,705 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-27T19:14:40,705 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-27T19:14:40,705 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:40,706 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-27T19:14:40,706 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:40,707 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f3 2024-11-27T19:14:40,707 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:40,707 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: mob 2024-11-27T19:14:40,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741865_1041 (size=49) 2024-11-27T19:14:40,767 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2024-11-27T19:14:41,116 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:14:41,116 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:41,117 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:41,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741866_1042 (size=1146) 2024-11-27T19:14:41,257 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T19:14:41,530 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:14:41,540 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:14:41,541 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734878409_default_test-1732734861815 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:41,543 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:14:41,543 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 }, snapshot procedure id = 34 2024-11-27T19:14:41,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1732734878409_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } in 3.1240 sec 2024-11-27T19:14:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T19:14:42,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732734861815 completed 2024-11-27T19:14:42,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } 2024-11-27T19:14:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734882585 (current time:1732734882585). 2024-11-27T19:14:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:14:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734882584_ns2_test-17327348618151 VERSION not specified, setting to 2 2024-11-27T19:14:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e932a95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:42,586 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:42,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:42,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:42,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77597aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:42,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:42,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:42,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:42,588 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:42,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1020f2ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:42,590 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:42,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:42,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:42,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:14:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:42,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:14:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:14:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } 2024-11-27T19:14:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-27T19:14:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:42,596 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:14:42,597 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:14:42,600 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:14:42,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741867_1043 (size=86) 2024-11-27T19:14:42,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2024-11-27T19:14:42,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:42,638 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2024-11-27T19:14:42,638 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2024-11-27T19:14:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:43,008 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:14:43,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6}] 2024-11-27T19:14:43,009 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:43,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2024-11-27T19:14:43,161 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:43,162 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 81ee8faa2e16b928d2ef19ee0c78f7f6 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2024-11-27T19:14:43,181 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/b15be9a8be414da1925a80e7914b3073 is 37, key is row10/f:q1/1732734865784/Put/seqid=0 2024-11-27T19:14:43,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741868_1044 (size=8413) 2024-11-27T19:14:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:43,587 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/b15be9a8be414da1925a80e7914b3073 2024-11-27T19:14:43,597 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/b15be9a8be414da1925a80e7914b3073 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 2024-11-27T19:14:43,605 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073, entries=99, sequenceid=103, filesize=8.2 K 2024-11-27T19:14:43,606 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 81ee8faa2e16b928d2ef19ee0c78f7f6 in 445ms, sequenceid=103, compaction requested=false 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 81ee8faa2e16b928d2ef19ee0c78f7f6: 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. for snapshot_1732734882584_ns2_test-17327348618151 completed. 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.' region-info for snapshot=snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073] hfiles 2024-11-27T19:14:43,606 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 for snapshot=snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741869_1045 (size=91) 2024-11-27T19:14:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:44,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:14:44,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-27T19:14:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-27T19:14:44,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734882584_ns2_test-17327348618151 on region 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:44,015 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:14:44,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-27T19:14:44,018 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:14:44,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 in 1.0080 sec 2024-11-27T19:14:44,019 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:14:44,020 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:14:44,020 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:44,021 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741870_1046 (size=456) 2024-11-27T19:14:44,437 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:14:44,445 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:14:44,445 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734882584_ns2_test-17327348618151 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:44,447 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:14:44,447 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 }, snapshot procedure id = 36 2024-11-27T19:14:44,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1732734882584_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } in 1.8530 sec 2024-11-27T19:14:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-27T19:14:44,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17327348618151 completed 2024-11-27T19:14:44,744 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732734871496 2024-11-27T19:14:44,744 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-27T19:14:44,746 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2024-11-27T19:14:44,747 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734871496_ns2:test-17327348618151 2024-11-27T19:14:44,748 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734882584_ns2_test-17327348618151 to hdfs://localhost:44785/backupUT/backup_1732734871496/ns2/test-17327348618151/ 2024-11-27T19:14:44,753 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:14:44,789 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:44,789 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734871496/ns2/test-17327348618151, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734871496/ns2/test-17327348618151/.hbase-snapshot/.tmp/snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:44,791 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:14:44,797 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734882584_ns2_test-17327348618151 to hdfs://localhost:44785/backupUT/backup_1732734871496/ns2/test-17327348618151/.hbase-snapshot/.tmp/snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:44,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741871_1047 (size=456) 2024-11-27T19:14:44,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741872_1048 (size=86) 2024-11-27T19:14:45,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:45,215 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:45,216 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,231 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-6167018200505079837.jar 2024-11-27T19:14:46,231 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,232 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,232 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,233 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,233 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,233 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,234 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,234 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,234 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:46,235 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:14:46,235 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:14:46,235 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:14:46,236 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:14:46,236 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:14:46,236 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:14:46,237 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:14:46,237 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:14:46,237 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:14:46,237 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:14:46,238 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:14:46,240 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:46,240 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:46,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:46,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:46,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:46,241 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:46,242 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:46,333 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:14:46,370 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734882584_ns2_test-17327348618151' hfile list 2024-11-27T19:14:46,376 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=8.2 K 2024-11-27T19:14:46,607 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1164841915_0001_2314533c-6b8c-47f0-b5ec-593a34601697/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:14:46,608 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1164841915_0001_2314533c-6b8c-47f0-b5ec-593a34601697/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:14:46,776 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-27T19:14:46,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741873_1049 (size=8413) 2024-11-27T19:14:47,296 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 output=hdfs://localhost:44785/backupUT/backup_1732734871496/ns2/test-17327348618151/archive/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 2024-11-27T19:14:47,299 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.016M/sec 2024-11-27T19:14:47,701 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:14:47,702 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:14:47,709 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:47,709 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734882584_ns2_test-17327348618151 finished. 2024-11-27T19:14:47,709 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734871496_test-1732734861815 2024-11-27T19:14:47,709 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734878409_default_test-1732734861815 to hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/ 2024-11-27T19:14:47,709 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:14:47,740 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:14:47,740 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/.hbase-snapshot/.tmp/snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:47,742 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:14:47,749 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734878409_default_test-1732734861815 to hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/.hbase-snapshot/.tmp/snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:47,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741874_1050 (size=84) 2024-11-27T19:14:47,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741875_1051 (size=1146) 2024-11-27T19:14:48,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:48,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:48,164 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-6195046649019367060.jar 2024-11-27T19:14:49,156 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,157 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,158 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:49,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:14:49,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:14:49,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:14:49,159 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:14:49,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:14:49,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:14:49,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:14:49,160 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:14:49,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:14:49,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:14:49,161 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:14:49,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:49,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:49,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:49,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:49,162 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:14:49,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:49,163 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:14:49,206 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:14:49,234 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734878409_default_test-1732734861815' hfile list 2024-11-27T19:14:49,236 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=18.8 K 2024-11-27T19:14:49,328 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1110752062_0002_f850842f-879b-449e-825d-73d4cfb920ea/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:14:49,328 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1110752062_0002_f850842f-879b-449e-825d-73d4cfb920ea/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:14:49,419 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-27T19:14:49,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741876_1052 (size=8413) 2024-11-27T19:14:49,864 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 output=hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:14:49,864 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-27T19:14:49,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741877_1053 (size=5484) 2024-11-27T19:14:49,937 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf output=hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf 2024-11-27T19:14:49,937 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.078M/sec 2024-11-27T19:14:50,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741878_1054 (size=5387) 2024-11-27T19:14:50,006 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 output=hdfs://localhost:44785/backupUT/backup_1732734871496/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 2024-11-27T19:14:50,006 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5387 (5.3 K) time=0sec 0.080M/sec 2024-11-27T19:14:50,386 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:14:50,387 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:14:50,395 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:50,395 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734878409_default_test-1732734861815 finished. 2024-11-27T19:14:50,395 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17327348618151 2024-11-27T19:14:50,395 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732734861815 2024-11-27T19:14:50,445 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734871496 2024-11-27T19:14:50,445 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-27T19:14:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741879_1055 (size=130) 2024-11-27T19:14:50,855 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44785/backupUT/backup_1732734871496/.backup.manifest 2024-11-27T19:14:50,857 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732734871496 finished: type=FULL,tablelist=ns2:test-17327348618151;test-1732734861815,targetRootDir=hdfs://localhost:44785/backupUT,startts=1732734878233,completets=1732734890443,bytescopied=0 2024-11-27T19:14:50,857 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-27T19:14:50,858 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:50,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734882584_ns2_test-17327348618151" type: DISABLED 2024-11-27T19:14:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734882584_ns2_test-17327348618151 2024-11-27T19:14:50,868 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734882584_ns2_test-17327348618151 for backup backup_1732734871496 succeeded. 2024-11-27T19:14:50,868 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:50,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734878409_default_test-1732734861815" type: DISABLED 2024-11-27T19:14:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734878409_default_test-1732734861815 2024-11-27T19:14:50,872 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734878409_default_test-1732734861815 for backup backup_1732734871496 succeeded. 2024-11-27T19:14:50,873 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-27T19:14:50,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-27T19:14:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-27T19:14:50,885 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-27T19:14:50,891 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-27T19:14:50,905 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732734871496 completed. 2024-11-27T19:14:50,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system', row='session:backup_1732734871496', locateType=CURRENT is [region=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:51,084 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(113): writing 99 rows to test-1732734861815 2024-11-27T19:14:51,105 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(116): written 99 rows to test-1732734861815 2024-11-27T19:14:51,276 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(120): written 111 rows to test-1732734861815 to Mob enabled CF 2024-11-27T19:14:51,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327348618151', row='row-t20', locateType=CURRENT is [region=ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:14:51,310 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(133): written 5 rows to ns2:test-17327348618151 2024-11-27T19:14:51,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$3(2313): Client=jenkins//172.17.0.2 split test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:14:51,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] assignment.SplitTableRegionProcedure(210): test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. splitKey isn't explicitly specified, will try to find a best split key from RS eb850308c015,41349,1732734853907 2024-11-27T19:14:51,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 because midkey is the same as first or last row 2024-11-27T19:14:51,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 because midkey is the same as first or last row 2024-11-27T19:14:51,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf because midkey is the same as first or last row 2024-11-27T19:14:51,324 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HRegion(2902): Flushing 44c0c476649e6e06a321127ef8d18631 3/3 column families, dataSize=11.48 KB heapSize=27.79 KB 2024-11-27T19:14:51,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/0a20a56274a94f0d8fb0e7ccc4372ef2 is 59, key is row-test-1732734861815-1-10/f:q1/1732734890942/Put/seqid=0 2024-11-27T19:14:51,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741880_1056 (size=10796) 2024-11-27T19:14:51,749 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/0a20a56274a94f0d8fb0e7ccc4372ef2 2024-11-27T19:14:51,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 is 63, key is row-test-1732734861815-3-100/mob:q1/1732734891261/Put/seqid=0 2024-11-27T19:14:51,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741881_1057 (size=5591) 2024-11-27T19:14:52,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:14:52,177 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 2024-11-27T19:14:52,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/50f81c27462241ef93eff9714ab5e5c3, store: [table=test-1732734861815 family=mob region=44c0c476649e6e06a321127ef8d18631] 2024-11-27T19:14:52,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/50f81c27462241ef93eff9714ab5e5c3 is 192, key is row-test-1732734861815-3-100/mob:q1/1732734891261/Put/seqid=0 2024-11-27T19:14:52,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741882_1058 (size=13760) 2024-11-27T19:14:52,585 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=6.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/50f81c27462241ef93eff9714ab5e5c3 2024-11-27T19:14:52,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f/0a20a56274a94f0d8fb0e7ccc4372ef2 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 2024-11-27T19:14:52,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2, entries=99, sequenceid=334, filesize=10.5 K 2024-11-27T19:14:52,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/mob/50f81c27462241ef93eff9714ab5e5c3 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 2024-11-27T19:14:52,607 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3, entries=111, sequenceid=334, filesize=13.4 K 2024-11-27T19:14:52,608 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.48 KB/11754, heapSize ~27.51 KB/28168, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 1285ms, sequenceid=334, compaction requested=false 2024-11-27T19:14:52,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.HRegion(2603): Flush status journal for 44c0c476649e6e06a321127ef8d18631: 2024-11-27T19:14:52,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 because midkey is the same as first or last row 2024-11-27T19:14:52,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 because midkey is the same as first or last row 2024-11-27T19:14:52,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 because midkey is the same as first or last row 2024-11-27T19:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=eb850308c015,41349,1732734853907 2024-11-27T19:14:52,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T19:14:52,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.CallRunner(138): callId: 303 service: MasterService methodName: SplitRegion size: 98 connection: 172.17.0.2:32916 deadline: 1732734951317, exception=org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row 2024-11-27T19:14:52,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2726): Operation: SPLIT_REGION, Table Name: default:test-1732734861815 failed with org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:52,616 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(144): region is not splittable, because java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.DoNotRetryIOException: org.apache.hadoop.hbase.DoNotRetryIOException: Region not splittable because bestSplitPoint = null, maybe table is too small for auto split. For force split, try specifying split row at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.checkSplittable(SplitTableRegionProcedure.java:240) at org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure.(SplitTableRegionProcedure.java:135) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.createSplitProcedure(AssignmentManager.java:1176) at org.apache.hadoop.hbase.master.HMaster$3.run(HMaster.java:2316) at org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil.submitProcedure(MasterProcedureUtil.java:137) at org.apache.hadoop.hbase.master.HMaster.splitRegion(HMaster.java:2309) at org.apache.hadoop.hbase.master.MasterRpcServices.splitRegion(MasterRpcServices.java:955) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:52,617 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T19:14:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732734861815,, stopping at row=test-1732734861815 ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,629 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732734861815,, stopping at row=test-1732734861815 ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,638 DEBUG [Time-limited test {}] backup.TestIncrementalBackup(149): split finished in =1327 2024-11-27T19:14:52,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,674 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-27T19:14:52,679 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734892638 2024-11-27T19:14:52,685 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732734871496} 2024-11-27T19:14:52,686 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-27T19:14:52,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:52,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734892731 (current time:1732734892731). 2024-11-27T19:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:14:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-27T19:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed733f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:52,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:52,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:52,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:52,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:52,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d5c1e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:52,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:52,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:52,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:52,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:52,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fce3035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:52,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:52,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:52,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:52,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:14:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:52,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:14:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-27T19:14:52,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:52,743 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:14:52,744 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:14:52,747 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:14:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741883_1059 (size=52) 2024-11-27T19:14:52,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:53,153 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:14:53,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56}] 2024-11-27T19:14:53,155 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:53,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2024-11-27T19:14:53,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:53,307 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=1.44 KB heapSize=2.75 KB 2024-11-27T19:14:53,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 is 158, key is trslm:hdfs://localhost:44785/backupUT\x00ns2:test-17327348618151/meta:log-roll-map/1732734890433/Put/seqid=0 2024-11-27T19:14:53,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741884_1060 (size=5914) 2024-11-27T19:14:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:53,730 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 2024-11-27T19:14:53,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/22887eff9aa5454db46c9e02a88f862c is 433, key is session:backup_1732734871496/session:context/1732734890890/Put/seqid=0 2024-11-27T19:14:53,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741885_1061 (size=5535) 2024-11-27T19:14:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:54,145 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/22887eff9aa5454db46c9e02a88f862c 2024-11-27T19:14:54,152 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 2024-11-27T19:14:54,158 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, entries=6, sequenceid=17, filesize=5.8 K 2024-11-27T19:14:54,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/22887eff9aa5454db46c9e02a88f862c as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c 2024-11-27T19:14:54,165 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, entries=2, sequenceid=17, filesize=5.4 K 2024-11-27T19:14:54,166 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~1.44 KB/1478, heapSize ~2.72 KB/2784, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 859ms, sequenceid=17, compaction requested=false 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2603): Flush status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. for snapshot_backup_system completed. 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.' region-info for snapshot=snapshot_backup_system 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9] hfiles 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 for snapshot=snapshot_backup_system 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c] hfiles 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 for snapshot=snapshot_backup_system 2024-11-27T19:14:54,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c for snapshot=snapshot_backup_system 2024-11-27T19:14:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741886_1062 (size=173) 2024-11-27T19:14:54,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:14:54,572 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-27T19:14:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-27T19:14:54,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:54,573 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=40, ppid=39, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:14:54,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-27T19:14:54,576 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:14:54,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 in 1.4210 sec 2024-11-27T19:14:54,577 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:14:54,578 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:14:54,578 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-27T19:14:54,579 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-27T19:14:54,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741887_1063 (size=810) 2024-11-27T19:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:54,990 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:14:54,997 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:14:54,998 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_backup_system 2024-11-27T19:14:54,999 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:14:54,999 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 39 2024-11-27T19:14:55,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2580 sec 2024-11-27T19:14:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-27T19:14:56,914 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-27T19:14:56,914 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732734892638 started at 1732734896914. 2024-11-27T19:14:56,916 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732734892638 has been started. 2024-11-27T19:14:56,921 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [test-1732734861815, ns2:test-17327348618151] 2024-11-27T19:14:56,924 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732734863107for backupID backup_1732734892638 2024-11-27T19:14:56,924 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-27T19:14:56,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-27T19:14:56,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-27T19:14:56,925 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-27T19:14:56,925 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-27T19:14:56,925 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-27T19:14:56,925 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-27T19:14:56,926 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:56,926 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:56,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:14:56,928 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:56,928 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:14:56,928 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:14:56,929 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:56,929 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-27T19:14:56,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:56,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:56,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-27T19:14:56,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:56,930 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-27T19:14:56,930 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-27T19:14:56,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-27T19:14:56,930 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-27T19:14:56,931 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-27T19:14:56,931 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-27T19:14:56,931 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-27T19:14:56,931 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure (rolllog) in zk 2024-11-27T19:14:56,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:56,933 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:56,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:56,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:56,933 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,933 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-27T19:14:56,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:56,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:56,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:56,934 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:56,935 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:56,935 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-27T19:14:56,935 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6ef9b3b4[Count = 0] remaining members to acquire global barrier 2024-11-27T19:14:56,935 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-27T19:14:56,935 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,938 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:56,939 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-27T19:14:56,939 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-27T19:14:56,939 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:56,939 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-27T19:14:56,939 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: eb850308c015,41349,1732734853907 2024-11-27T19:14:56,939 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732734878286 highest: 1732734878306 on eb850308c015,41349,1732734853907 2024-11-27T19:14:56,939 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0:(num 1732734878286) roll requested 2024-11-27T19:14:56,953 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939, exclude list is [], retry=0 2024-11-27T19:14:56,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:56,958 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 2024-11-27T19:14:56,958 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:56,958 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 is not closed yet, will try archiving it next time 2024-11-27T19:14:56,959 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1:(num 1732734878306) roll requested 2024-11-27T19:14:56,960 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 not finished, retry = 0 2024-11-27T19:14:56,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741857_1033 (size=1198) 2024-11-27T19:14:56,972 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959, exclude list is [], retry=0 2024-11-27T19:14:56,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:56,976 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 with entries=224, filesize=29.77 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 2024-11-27T19:14:56,976 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:56,976 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 is not closed yet, will try archiving it next time 2024-11-27T19:14:56,976 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.meta:.meta(num 1732734878325) roll requested 2024-11-27T19:14:56,976 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734863107 2024-11-27T19:14:56,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741858_1034 (size=30490) 2024-11-27T19:14:56,977 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 not finished, retry = 0 2024-11-27T19:14:56,990 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta, exclude list is [], retry=0 2024-11-27T19:14:56,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:14:56,994 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta 2024-11-27T19:14:56,994 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:14:56,994 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta is not closed yet, will try archiving it next time 2024-11-27T19:14:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741859_1035 (size=93) 2024-11-27T19:14:56,996 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta not finished, retry = 0 2024-11-27T19:14:57,000 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2024-11-27T19:14:57,000 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732734896939 on eb850308c015,41349,1732734853907 2024-11-27T19:14:57,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:14:57,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:14:57,018 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-27T19:14:57,019 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'eb850308c015,41349,1732734853907' in zk 2024-11-27T19:14:57,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,021 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-27T19:14:57,021 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:14:57,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:57,021 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-27T19:14:57,021 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:57,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:57,022 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:57,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:57,023 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:57,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:57,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:57,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:57,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'eb850308c015,41349,1732734853907': 2024-11-27T19:14:57,026 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-27T19:14:57,026 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-27T19:14:57,026 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'eb850308c015,41349,1732734853907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-27T19:14:57,026 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-27T19:14:57,026 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-27T19:14:57,026 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-27T19:14:57,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:14:57,029 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:14:57,029 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:14:57,030 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:14:57,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:14:57,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:14:57,030 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:57,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:14:57,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:57,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:57,032 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:14:57,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:14:57,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:14:57,035 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:57,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:14:57,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:14:57,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:14:57,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:14:57,036 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:14:57,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-27T19:14:57,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-27T19:14:57,036 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:14:57,036 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:14:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-27T19:14:57,045 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {eb850308c015:41349=1732734863107} newestTimestamps: {eb850308c015:41349=1732734878306} 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 2024-11-27T19:14:57,048 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 2024-11-27T19:14:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741891_1067 (size=364) 2024-11-27T19:14:57,097 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta 2024-11-27T19:14:57,403 DEBUG [master/eb850308c015:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732734861815 2024-11-27T19:14:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741892_1068 (size=950) 2024-11-27T19:14:57,461 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17327348618151 target: hdfs://localhost:44785/backupUT/backup_1732734892638/ns2/test-17327348618151 descriptor: 'ns2:test-17327348618151', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:57,461 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:14:57,463 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17327348618151 2024-11-27T19:14:57,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741893_1069 (size=50) 2024-11-27T19:14:57,816 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000002.908 2024-11-27T19:14:57,816 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000003.950 2024-11-27T19:14:57,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732734861815 in 417 msec 2024-11-27T19:14:57,869 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17327348618151 2024-11-27T19:14:57,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741894_1070 (size=950) 2024-11-27T19:14:58,281 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732734861815 target: hdfs://localhost:44785/backupUT/backup_1732734892638/default/test-1732734861815 descriptor: 'test-1732734861815', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:14:58,281 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:14:58,284 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732734861815 2024-11-27T19:14:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741895_1071 (size=53) 2024-11-27T19:14:58,690 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732734861815 2024-11-27T19:14:58,700 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638 from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286;hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 2024-11-27T19:14:58,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ceaa52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:58,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:14:58,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:14:58,703 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:14:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:14:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:14:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107dcc5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:58,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:14:58,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:14:58,704 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:14:58,705 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:14:58,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46015b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:14:58,705 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T19:14:58,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is eb850308c015,39035,1732734852191 2024-11-27T19:14:58,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78a01db 2024-11-27T19:14:58,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T19:14:58,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T19:14:58,713 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-27T19:14:58,713 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732734861815 2024-11-27T19:14:58,713 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732734861815,, stopping at row=test-1732734861815 ,, for max=2147483647 with caching=100 2024-11-27T19:14:58,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:14:58,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:14:58,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:14:58,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:14:58,717 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732734861815: test-1732734861815; 2024-11-27T19:14:58,717 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17327348618151 2024-11-27T19:14:58,718 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17327348618151,, stopping at row=ns2:test-17327348618151 ,, for max=2147483647 with caching=100 2024-11-27T19:14:58,719 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17327348618151: ns2:test-17327348618151; 2024-11-27T19:14:58,719 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 2 reduce partitions to match current region count for all tables 2024-11-27T19:14:58,720 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/hbase-staging/partitions_f4d7ce37-5f16-4fd2-9941-f535157adccd 2024-11-27T19:14:58,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741896_1072 (size=192) 2024-11-27T19:14:59,143 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:59,144 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:14:59,144 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,129 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-9809388719887174446.jar 2024-11-27T19:15:00,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,130 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,131 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,132 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:00,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:00,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:00,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:00,133 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:00,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:00,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:00,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:00,134 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:00,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:00,135 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:00,136 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,137 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:00,138 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:00,138 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: default:test-1732734861815,ns2:test-17327348618151 2024-11-27T19:15:00,138 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:15:00,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:154) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:00,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:00,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:00,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:00,139 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:01,121 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-4544434222750215747.jar 2024-11-27T19:15:01,135 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:01,162 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 for WAL files 2024-11-27T19:15:01,163 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 2024-11-27T19:15:01,163 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 for WAL files 2024-11-27T19:15:01,164 INFO [Time-limited test {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 2024-11-27T19:15:01,264 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local508431551_0003_0ca73545-1297-4853-a9e1-0336a323f844/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:01,264 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local508431551_0003_0ca73545-1297-4853-a9e1-0336a323f844/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:01,362 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 for hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 (-9223372036854775808:9223372036854775807) length:30490 2024-11-27T19:15:01,386 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-27T19:15:01,387 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-27T19:15:01,441 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 for hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 (-9223372036854775808:9223372036854775807) length:1198 2024-11-27T19:15:01,444 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2024-11-27T19:15:01,445 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2024-11-27T19:15:01,529 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcc4288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,529 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:01,530 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:01,531 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:01,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:01,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:01,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5532c175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:01,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:01,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,533 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:01,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654cbaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,535 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:01,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:01,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:01,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:01,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'ns2:test-17327348618151', row='row-t20', locateType=CURRENT is [region=ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:15:01,539 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-27T19:15:01,539 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:01,539 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,539 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:01,540 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-t20] 2024-11-27T19:15:01,540 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: eb850308c015 2024-11-27T19:15:01,546 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638/_temporary/0/_temporary/attempt_local508431551_0003_r_000000_0/ns2/test-17327348618151/f/a942bc803fcb40fab0afb363644eacfc is 38, key is row-t20/f:q1/1732734891296/Put/seqid=0 2024-11-27T19:15:01,548 WARN [Thread-950 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [eb850308c015:41349] Specified favored nodes: [eb850308c015:41349] 2024-11-27T19:15:01,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741897_1073 (size=5274) 2024-11-27T19:15:01,978 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bc6408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,978 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:01,979 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:01,979 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ee7332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,981 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:01,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f2af5aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,982 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:01,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:01,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:01,984 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:01,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732734861815', row='row-test-1732734861815-1-0', locateType=CURRENT is [region=test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., hostname=eb850308c015,41349,1732734853907, seqNum=107] 2024-11-27T19:15:01,986 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-27T19:15:01,986 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:01,986 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,986 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:01,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:01,986 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732734861815-1-0] 2024-11-27T19:15:01,987 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: eb850308c015 2024-11-27T19:15:01,997 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7babd5d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,997 DEBUG [pool-205-thread-1 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:01,998 DEBUG [pool-205-thread-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:01,998 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbe970f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:01,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:02,000 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:02,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bf257c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:02,001 DEBUG [pool-205-thread-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:02,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:02,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:02,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:02,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test-1732734861815', row='row-test-1732734861815-3-0', locateType=CURRENT is [region=test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., hostname=eb850308c015,41349,1732734853907, seqNum=107] 2024-11-27T19:15:02,005 INFO [pool-205-thread-1 {}] client.AsyncConnectionImpl(233): Connection has been closed by pool-205-thread-1. 2024-11-27T19:15:02,005 DEBUG [pool-205-thread-1 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:299) at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2$1.write(HFileOutputFormat2.java:234) at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:559) at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89) at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:48) at org.apache.hadoop.hbase.mapreduce.CellSortReducer.reduce(CellSortReducer.java:35) at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171) at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:628) at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:390) at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:347) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:02,005 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:02,005 DEBUG [pool-205-thread-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:02,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:02,005 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(308): First rowkey: [row-test-1732734861815-3-0] 2024-11-27T19:15:02,005 DEBUG [pool-205-thread-1 {}] mapreduce.HFileOutputFormat2$1(314): Use favored nodes writer: eb850308c015 2024-11-27T19:15:02,016 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638/_temporary/0/_temporary/attempt_local508431551_0003_r_000001_0/default/test-1732734861815/f/3bf351b72fdc4192b3182f93b356c467 is 59, key is row-test-1732734861815-1-10/f:q1/1732734890942/Put/seqid=0 2024-11-27T19:15:02,018 WARN [Thread-962 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [eb850308c015:41349] Specified favored nodes: [eb850308c015:41349] 2024-11-27T19:15:02,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741898_1074 (size=10769) 2024-11-27T19:15:02,421 DEBUG [pool-205-thread-1 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638/_temporary/0/_temporary/attempt_local508431551_0003_r_000001_0/default/test-1732734861815/mob/126120b5fad24362bce9de3b9fb1d670 is 63, key is row-test-1732734861815-3-100/mob:q1/1732734891261/Put/seqid=0 2024-11-27T19:15:02,423 WARN [Thread-964 {}] hdfs.DataStreamer(2024): These favored nodes were specified but not chosen: [eb850308c015:41349] Specified favored nodes: [eb850308c015:41349] 2024-11-27T19:15:02,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741899_1075 (size=11810) 2024-11-27T19:15:03,307 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(344): Incremental copy HFiles is starting. dest=hdfs://localhost:44785/backupUT 2024-11-27T19:15:03,308 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(354): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1732734892638 2024-11-27T19:15:03,308 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(385): Doing COPY_TYPE_DISTCP 2024-11-27T19:15:03,342 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(394): DistCp options: [hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638, hdfs://localhost:44785/backupUT] 2024-11-27T19:15:03,420 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:03,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741900_1076 (size=11810) 2024-11-27T19:15:03,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741901_1077 (size=5274) 2024-11-27T19:15:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741902_1078 (size=10769) 2024-11-27T19:15:05,040 INFO [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(224): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2024-11-27T19:15:05,043 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(144): Backup progress data "100%" has been updated to backup system table for backup_1732734892638 2024-11-27T19:15:05,043 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(232): Backup progress data updated to backup system table: "Progress: 100.0% - 27853 bytes copied." 2024-11-27T19:15:05,043 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(242): DistCp job-id: job_local1791629025_0004 completed: true true 2024-11-27T19:15:05,044 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(245): Counters: 26 File System Counters FILE: Number of bytes read=3825260 FILE: Number of bytes written=6211310 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=231464 HDFS: Number of bytes written=2433179 HDFS: Number of read operations=643 HDFS: Number of large read operations=0 HDFS: Number of write operations=249 HDFS: Number of bytes read erasure-coded=0 Map-Reduce Framework Map input records=12 Map output records=0 Input split bytes=340 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1048576000 File Input Format Counters Bytes Read=3004 File Output Format Counters Bytes Written=0 DistCp Counters Bandwidth in Bytes=27853 Bytes Copied=27853 Bytes Expected=27853 Files Copied=4 DIR_COPY=8 2024-11-27T19:15:05,044 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(365): Incremental copy HFiles from hdfs://localhost:44785/backupUT/.tmp/backup_1732734892638 to hdfs://localhost:44785/backupUT finished. 2024-11-27T19:15:05,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'backup:system_bulk', row='bulk:ns2:test-17327348618151:', locateType=CURRENT is [region=backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc., hostname=eb850308c015,41349,1732734853907, seqNum=2] 2024-11-27T19:15:05,059 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734892638 2024-11-27T19:15:05,063 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732734871496} 2024-11-27T19:15:05,063 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-27T19:15:05,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741903_1079 (size=384) 2024-11-27T19:15:05,468 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44785/backupUT/backup_1732734892638/.backup.manifest 2024-11-27T19:15:05,469 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732734892638 finished: type=INCREMENTAL,tablelist=ns2:test-17327348618151;test-1732734861815,targetRootDir=hdfs://localhost:44785/backupUT,startts=1732734896914,completets=1732734905059,bytescopied=0 2024-11-27T19:15:05,469 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-27T19:15:05,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-27T19:15:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-27T19:15:05,477 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-27T19:15:05,479 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-27T19:15:05,483 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732734892638 completed. 2024-11-27T19:15:05,483 DEBUG [Time-limited test {}] impl.BackupSystemTable(458): deleted 0 original bulkload rows 2024-11-27T19:15:05,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:05,499 DEBUG [Time-limited test {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:44785/backupUT/backup_1732734892638/.backup.manifest 2024-11-27T19:15:05,501 DEBUG [Time-limited test {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1732734892638/.backup.manifest 2024-11-27T19:15:05,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table test-1732734861815 from 'test-1732734861815', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1732734861815', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=test-1732734861815 2024-11-27T19:15:05,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741904_1080 (size=908) 2024-11-27T19:15:05,918 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000003.950 2024-11-27T19:15:05,918 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000004.908 2024-11-27T19:15:05,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=test-1732734861815}] 2024-11-27T19:15:05,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE}] 2024-11-27T19:15:05,924 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE 2024-11-27T19:15:05,925 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=CLOSING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:05,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE because future has completed 2024-11-27T19:15:05,927 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T19:15:05,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:15:06,080 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(122): Close 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,080 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T19:15:06,080 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1722): Closing 44c0c476649e6e06a321127ef8d18631, disabling compactions & flushes 2024-11-27T19:15:06,080 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1755): Closing region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,081 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,081 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1843): Acquired close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. after waiting 0 ms 2024-11-27T19:15:06,081 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1853): Updates disabled for region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,086 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=106 2024-11-27T19:15:06,087 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:06,087 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1973): Closed test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,087 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegion(1676): Region close journal for 44c0c476649e6e06a321127ef8d18631: Waiting for close lock at 1732734906080Running coprocessor pre-close hooks at 1732734906080Disabling compacts and flushes for region at 1732734906080Disabling writes for close at 1732734906081 (+1 ms)Writing region close event to WAL at 1732734906082 (+1 ms)Running coprocessor post-close hooks at 1732734906087 (+5 ms)Closed at 1732734906087 2024-11-27T19:15:06,087 WARN [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] regionserver.HRegionServer(3299): Not adding moved region record: 44c0c476649e6e06a321127ef8d18631 to self. 2024-11-27T19:15:06,089 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=45}] handler.UnassignRegionHandler(157): Closed 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,089 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=CLOSED 2024-11-27T19:15:06,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:15:06,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T19:15:06,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 in 164 msec 2024-11-27T19:15:06,094 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE; state=CLOSED, location=eb850308c015,41349,1732734853907; forceNewPlan=false, retain=true 2024-11-27T19:15:06,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:06,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE because future has completed 2024-11-27T19:15:06,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:15:06,403 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,403 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c0c476649e6e06a321127ef8d18631, NAME => 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:15:06,403 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:15:06,403 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1732734861815 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,404 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:15:06,404 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,404 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,405 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,406 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName f 2024-11-27T19:15:06,406 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:06,413 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 2024-11-27T19:15:06,417 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:15:06,417 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:15:06,417 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f2 of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,419 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName f2 2024-11-27T19:15:06,419 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:06,420 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/f2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:15:06,420 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family mob of region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,421 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c0c476649e6e06a321127ef8d18631 columnFamilyName mob 2024-11-27T19:15:06,422 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:06,428 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 2024-11-27T19:15:06,432 DEBUG [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf 2024-11-27T19:15:06,432 INFO [StoreOpener-44c0c476649e6e06a321127ef8d18631-1 {}] regionserver.HStore(327): Store=44c0c476649e6e06a321127ef8d18631/mob, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:15:06,433 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,433 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,435 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,436 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,436 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,437 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1732734861815 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-27T19:15:06,438 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,439 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 44c0c476649e6e06a321127ef8d18631; next sequenceid=338; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10435246400, jitterRate=-0.028141945600509644}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-27T19:15:06,439 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:06,441 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 44c0c476649e6e06a321127ef8d18631: Running coprocessor pre-open hook at 1732734906404Writing region info on filesystem at 1732734906404Initializing all the Stores at 1732734906405 (+1 ms)Instantiating store for column family {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734906405Instantiating store for column family {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734906405Instantiating store for column family {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734906405Cleaning up temporary data from old regions at 1732734906436 (+31 ms)Running coprocessor post-open hooks at 1732734906439 (+3 ms)Region opened successfully at 1732734906441 (+2 ms) 2024-11-27T19:15:06,442 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., pid=46, masterSystemTime=1732734906399 2024-11-27T19:15:06,444 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,444 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:06,445 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=44c0c476649e6e06a321127ef8d18631, regionState=OPEN, openSeqNum=338, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:06,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:15:06,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-27T19:15:06,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c0c476649e6e06a321127ef8d18631, server=eb850308c015,41349,1732734853907 in 200 msec 2024-11-27T19:15:06,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-27T19:15:06,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test-1732734861815, region=44c0c476649e6e06a321127ef8d18631, REOPEN/MOVE in 527 msec 2024-11-27T19:15:06,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T19:15:06,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=test-1732734861815 in 531 msec 2024-11-27T19:15:06,456 DEBUG [PEWorker-5 {}] procedure.MasterDDLOperationHelper(51): Removing family=f3 from table=test-1732734861815 2024-11-27T19:15:06,462 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f3/342aba8c72f746219de51bde2e2ec310 2024-11-27T19:15:06,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; ModifyTableProcedure table=test-1732734861815 in 959 msec 2024-11-27T19:15:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T19:15:06,466 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY, Table Name: default:test-1732734861815 completed 2024-11-27T19:15:06,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,497 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-27T19:15:06,501 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734906466 2024-11-27T19:15:06,503 DEBUG [Time-limited test {}] impl.TableBackupClient(330): Dependent incremental backup image: {BackupID=backup_1732734892638} 2024-11-27T19:15:06,503 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732734871496} 2024-11-27T19:15:06,503 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 2 ancestors for the current backup. 2024-11-27T19:15:06,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,543 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected Exception : Mismatch in column family descriptor for table: test-1732734861815 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException: Mismatch in column family descriptor for table: test-1732734861815 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} Backup families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at org.apache.hadoop.hbase.backup.impl.ColumnFamilyMismatchException$ColumnFamilyMismatchExceptionBuilder.build(ColumnFamilyMismatchException.java:56) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.verifyCfCompatibility(IncrementalTableBackupClient.java:513) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:280) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.lambda$TestIncBackupRestore$0(TestIncrementalBackup.java:171) ~[test-classes/:?] at org.junit.Assert.assertThrows(Assert.java:1001) ~[junit-4.13.2.jar:4.13.2] at org.junit.Assert.assertThrows(Assert.java:981) ~[junit-4.13.2.jar:4.13.2] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:170) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T19:15:06,546 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732734906466,startts=0,failedts=1732734906544,failedphase=REQUEST,failedmessage= Mismatch in column family descriptor for table: test-1732734861815 Current families: {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'},{NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'tru 2024-11-27T19:15:06,546 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-27T19:15:06,548 WARN [Time-limited test {}] impl.BackupSystemTable(1549): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2024-11-27T19:15:06,548 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-27T19:15:06,550 ERROR [Time-limited test {}] impl.BackupSystemTable(1578): Snapshot snapshot_backup_system does not exists 2024-11-27T19:15:06,550 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: REQUEST 2024-11-27T19:15:06,552 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-27T19:15:06,554 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732734906466 failed. 2024-11-27T19:15:06,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:06,571 INFO [Time-limited test {}] impl.BackupManager(225): Full backup all the tables available in the cluster: [backup:system_bulk, test-1732734861815, ns2:test-17327348618151, ns3:test-17327348618152, ns4:test-17327348618153] 2024-11-27T19:15:06,572 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-27T19:15:06,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734906576 (current time:1732734906576). 2024-11-27T19:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-27T19:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb207b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:06,577 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142a5173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:06,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:06,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:06,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e1788b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:06,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:06,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:06,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:06,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:06,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-27T19:15:06,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:06,586 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:06,587 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:06,589 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:06,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741905_1081 (size=52) 2024-11-27T19:15:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:06,996 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:06,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56}] 2024-11-27T19:15:06,997 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:07,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=48 2024-11-27T19:15:07,150 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:07,150 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=2.61 KB heapSize=3.99 KB 2024-11-27T19:15:07,154 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/d3b2e5a100664b3cac810fb2b372a8df is 158, key is trslm:hdfs://localhost:44785/backupUT\x00ns2:test-17327348618151/meta:log-roll-map/1732734905049/Put/seqid=0 2024-11-27T19:15:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741906_1082 (size=5745) 2024-11-27T19:15:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:07,382 DEBUG [master/eb850308c015:0.Chore.1 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732734861815 2024-11-27T19:15:07,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741907_1083 (size=950) 2024-11-27T19:15:07,559 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=492 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/d3b2e5a100664b3cac810fb2b372a8df 2024-11-27T19:15:07,569 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/e59036f545b9458eadcf9f12163ddeee is 1245, key is session:backup_1732734906466/session:context/1732734906551/Put/seqid=0 2024-11-27T19:15:07,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741908_1084 (size=6682) 2024-11-27T19:15:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:07,793 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000004.908 2024-11-27T19:15:07,793 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/.tabledesc/.tableinfo.0000000005.950 2024-11-27T19:15:07,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; InitializeStoreFileTrackerProcedure table=test-1732734861815 in 412 msec 2024-11-27T19:15:07,974 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.13 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/e59036f545b9458eadcf9f12163ddeee 2024-11-27T19:15:07,981 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/d3b2e5a100664b3cac810fb2b372a8df as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df 2024-11-27T19:15:07,986 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df, entries=4, sequenceid=31, filesize=5.6 K 2024-11-27T19:15:07,987 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/e59036f545b9458eadcf9f12163ddeee as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee 2024-11-27T19:15:07,993 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee, entries=3, sequenceid=31, filesize=6.5 K 2024-11-27T19:15:07,994 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(3140): Finished flush of dataSize ~2.61 KB/2669, heapSize ~3.96 KB/4056, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 843ms, sequenceid=31, compaction requested=true 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.HRegion(2603): Flush status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. for snapshot_backup_system completed. 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.' region-info for snapshot=snapshot_backup_system 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df] hfiles 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 for snapshot=snapshot_backup_system 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df for snapshot=snapshot_backup_system 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee] hfiles 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 for snapshot=snapshot_backup_system 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c for snapshot=snapshot_backup_system 2024-11-27T19:15:07,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee for snapshot=snapshot_backup_system 2024-11-27T19:15:07,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741909_1085 (size=251) 2024-11-27T19:15:08,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:08,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-27T19:15:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-27T19:15:08,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:08,401 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:08,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-27T19:15:08,405 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:08,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 in 1.4070 sec 2024-11-27T19:15:08,405 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:08,406 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:08,406 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-27T19:15:08,407 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-27T19:15:08,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741910_1086 (size=888) 2024-11-27T19:15:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:08,818 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:08,827 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:08,827 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_backup_system 2024-11-27T19:15:08,828 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:08,829 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2024-11-27T19:15:08,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2450 sec 2024-11-27T19:15:09,762 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 81ee8faa2e16b928d2ef19ee0c78f7f6, had cached 0 bytes from a total of 8413 2024-11-27T19:15:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-11-27T19:15:10,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-27T19:15:10,764 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732734906555 started at 1732734910763. 2024-11-27T19:15:10,766 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732734906555 has been started. 2024-11-27T19:15:10,769 INFO [Time-limited test {}] impl.FullTableBackupClient(153): Execute roll log procedure for full backup ... 2024-11-27T19:15:10,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-27T19:15:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-27T19:15:10,770 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-27T19:15:10,770 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-27T19:15:10,770 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-27T19:15:10,770 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-27T19:15:10,771 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,771 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:10,774 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:15:10,774 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:15:10,774 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:15:10,775 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,775 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-27T19:15:10,775 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:10,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-27T19:15:10,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:10,776 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-27T19:15:10,777 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-27T19:15:10,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-27T19:15:10,777 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-27T19:15:10,777 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-27T19:15:10,777 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-27T19:15:10,777 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-27T19:15:10,777 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure (rolllog) in zk 2024-11-27T19:15:10,779 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:10,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:10,780 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,780 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-27T19:15:10,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:10,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:10,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:10,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:10,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-27T19:15:10,782 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4528fd2c[Count = 0] remaining members to acquire global barrier 2024-11-27T19:15:10,782 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-27T19:15:10,782 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,784 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,784 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-27T19:15:10,784 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,784 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-27T19:15:10,784 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-27T19:15:10,784 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: eb850308c015,41349,1732734853907 2024-11-27T19:15:10,784 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732734896939 highest: 1732734896959 on eb850308c015,41349,1732734853907 2024-11-27T19:15:10,784 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0:(num 1732734896939) roll requested 2024-11-27T19:15:10,798 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784, exclude list is [], retry=0 2024-11-27T19:15:10,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:10,803 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 2024-11-27T19:15:10,803 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:10,803 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 is not closed yet, will try archiving it next time 2024-11-27T19:15:10,803 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1:(num 1732734896959) roll requested 2024-11-27T19:15:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741888_1064 (size=93) 2024-11-27T19:15:10,804 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 not finished, retry = 0 2024-11-27T19:15:10,817 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803, exclude list is [], retry=0 2024-11-27T19:15:10,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:10,821 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 with entries=15, filesize=4.80 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803 2024-11-27T19:15:10,821 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:10,821 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 is not closed yet, will try archiving it next time 2024-11-27T19:15:10,821 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.meta:.meta(num 1732734896976) roll requested 2024-11-27T19:15:10,821 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734878306 2024-11-27T19:15:10,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741889_1065 (size=4927) 2024-11-27T19:15:10,835 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta, exclude list is [], retry=0 2024-11-27T19:15:10,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:10,839 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta with entries=4, filesize=1.85 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta 2024-11-27T19:15:10,839 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:10,839 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta is not closed yet, will try archiving it next time 2024-11-27T19:15:10,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741890_1066 (size=1903) 2024-11-27T19:15:10,841 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta not finished, retry = 0 2024-11-27T19:15:10,845 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-27T19:15:10,845 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732734910784 on eb850308c015,41349,1732734853907 2024-11-27T19:15:10,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:10,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:10,864 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-27T19:15:10,864 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'eb850308c015,41349,1732734853907' in zk 2024-11-27T19:15:10,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,866 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-27T19:15:10,866 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:15:10,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,866 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-27T19:15:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:10,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:10,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,868 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:10,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:10,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,869 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:10,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'eb850308c015,41349,1732734853907': 2024-11-27T19:15:10,870 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-27T19:15:10,870 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'eb850308c015,41349,1732734853907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-27T19:15:10,870 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-27T19:15:10,870 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-27T19:15:10,870 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-27T19:15:10,870 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-27T19:15:10,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:15:10,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,874 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:10,874 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:15:10,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:15:10,875 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:10,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:10,877 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:10,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:10,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:10,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:10,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:10,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:15:10,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:10,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:10,881 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:15:10,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:15:10,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:15:10,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:10,881 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:15:10,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-27T19:15:10,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-27T19:15:10,882 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:15:10,882 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:15:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-27T19:15:10,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-27T19:15:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734910897 (current time:1732734910897). 2024-11-27T19:15:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734910896_backup_system_bulk VERSION not specified, setting to 2 2024-11-27T19:15:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b65326e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:10,899 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:10,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:10,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:10,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a9faf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:10,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:10,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:10,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:10,901 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:10,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd0bf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:10,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:10,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:10,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:10,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:10,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:10,906 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734896939 2024-11-27T19:15:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } 2024-11-27T19:15:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-27T19:15:10,908 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:10,909 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:10,911 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:10,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741914_1090 (size=76) 2024-11-27T19:15:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:11,258 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T19:15:11,318 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:11,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc}] 2024-11-27T19:15:11,319 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:15:11,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 05e9d065d2547a02238ecbaeb2f3acbc: 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. for snapshot_1732734910896_backup_system_bulk completed. 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(241): Storing 'backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc.' region-info for snapshot=snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:11,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:15:11,472 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:15:11,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741915_1091 (size=61) 2024-11-27T19:15:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:11,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-27T19:15:11,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:11,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T19:15:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T19:15:11,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734910896_backup_system_bulk on region 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:15:11,879 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=50, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:15:11,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T19:15:11,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 05e9d065d2547a02238ecbaeb2f3acbc in 562 msec 2024-11-27T19:15:11,881 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:11,882 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:11,883 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:11,883 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:11,883 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:11,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741916_1092 (size=702) 2024-11-27T19:15:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:12,294 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:12,299 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:12,299 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734910896_backup_system_bulk to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:12,301 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:12,301 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 }, snapshot procedure id = 50 2024-11-27T19:15:12,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_1732734910896_backup_system_bulk table=backup:system_bulk type=FLUSH ttl=0 } in 1.3950 sec 2024-11-27T19:15:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T19:15:13,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system_bulk completed 2024-11-27T19:15:13,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } 2024-11-27T19:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734913054 (current time:1732734913054). 2024-11-27T19:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734913053_ns2_test-17327348618151 VERSION not specified, setting to 2 2024-11-27T19:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9227332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:13,055 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4f6990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:13,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:13,057 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:13,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c73106, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:13,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:13,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:13,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:13,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:13,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } 2024-11-27T19:15:13,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-27T19:15:13,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:13,063 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:13,064 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:13,066 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:13,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741917_1093 (size=86) 2024-11-27T19:15:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:13,472 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:13,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6}] 2024-11-27T19:15:13,473 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:15:13,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2024-11-27T19:15:13,625 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:13,625 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 81ee8faa2e16b928d2ef19ee0c78f7f6 1/1 column families, dataSize=170 B heapSize=816 B 2024-11-27T19:15:13,630 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/aaa603a48f7c48c7910eb878894c8d3b is 38, key is row-t20/f:q1/1732734891296/Put/seqid=0 2024-11-27T19:15:13,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741918_1094 (size=5134) 2024-11-27T19:15:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:14,034 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/aaa603a48f7c48c7910eb878894c8d3b 2024-11-27T19:15:14,041 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/.tmp/f/aaa603a48f7c48c7910eb878894c8d3b as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b 2024-11-27T19:15:14,046 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b, entries=5, sequenceid=111, filesize=5.0 K 2024-11-27T19:15:14,047 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for 81ee8faa2e16b928d2ef19ee0c78f7f6 in 422ms, sequenceid=111, compaction requested=false 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 81ee8faa2e16b928d2ef19ee0c78f7f6: 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. for snapshot_1732734913053_ns2_test-17327348618151 completed. 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(241): Storing 'ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6.' region-info for snapshot=snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b] hfiles 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 for snapshot=snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,048 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b for snapshot=snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741919_1095 (size=130) 2024-11-27T19:15:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:14,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:14,453 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T19:15:14,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T19:15:14,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734913053_ns2_test-17327348618151 on region 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:15:14,454 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=53, ppid=52, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:15:14,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T19:15:14,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 81ee8faa2e16b928d2ef19ee0c78f7f6 in 983 msec 2024-11-27T19:15:14,457 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:14,458 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:14,458 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:14,458 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,459 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741920_1096 (size=496) 2024-11-27T19:15:14,869 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:14,875 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:14,876 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734913053_ns2_test-17327348618151 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:14,877 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:14,877 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 }, snapshot procedure id = 52 2024-11-27T19:15:14,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1732734913053_ns2_test-17327348618151 table=ns2:test-17327348618151 type=FLUSH ttl=0 } in 1.8160 sec 2024-11-27T19:15:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T19:15:15,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns2:test-17327348618151 completed 2024-11-27T19:15:15,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } 2024-11-27T19:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734915215 (current time:1732734915215). 2024-11-27T19:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734915214_ns3_test-17327348618152 VERSION not specified, setting to 2 2024-11-27T19:15:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab89678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:15,217 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:15,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:15,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:15,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed21bc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:15,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:15,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:15,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:15,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:15,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19deadde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:15,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:15,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:15,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:15,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:15,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } 2024-11-27T19:15:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-27T19:15:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:15,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:15,225 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:15,227 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:15,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741921_1097 (size=86) 2024-11-27T19:15:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:15,633 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:15,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6}] 2024-11-27T19:15:15,634 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:15:15,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2024-11-27T19:15:15,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:15,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for cf00efa8e14baf8163e951bfbb0a60c6: 2024-11-27T19:15:15,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. for snapshot_1732734915214_ns3_test-17327348618152 completed. 2024-11-27T19:15:15,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(241): Storing 'ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6.' region-info for snapshot=snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:15,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:15,787 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:15:15,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741922_1098 (size=52) 2024-11-27T19:15:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:16,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:16,193 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T19:15:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T19:15:16,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734915214_ns3_test-17327348618152 on region cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:15:16,194 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=55, ppid=54, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:15:16,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T19:15:16,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; SnapshotRegionProcedure cf00efa8e14baf8163e951bfbb0a60c6 in 562 msec 2024-11-27T19:15:16,196 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:16,197 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:16,198 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:16,198 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:16,198 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:16,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741923_1099 (size=443) 2024-11-27T19:15:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:16,609 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:16,614 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:16,614 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734915214_ns3_test-17327348618152 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:16,616 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:16,616 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 }, snapshot procedure id = 54 2024-11-27T19:15:16,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_1732734915214_ns3_test-17327348618152 table=ns3:test-17327348618152 type=FLUSH ttl=0 } in 1.3930 sec 2024-11-27T19:15:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T19:15:17,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns3:test-17327348618152 completed 2024-11-27T19:15:17,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } 2024-11-27T19:15:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734917374 (current time:1732734917374). 2024-11-27T19:15:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734917373_default_test-1732734861815 VERSION not specified, setting to 2 2024-11-27T19:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29857319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:17,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:17,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:17,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:17,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecca3de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:17,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:17,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:17,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:17,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:17,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240de417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:17,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:17,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:17,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:17,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:17,382 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } 2024-11-27T19:15:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-27T19:15:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:17,384 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:17,385 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:17,387 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:17,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741924_1100 (size=84) 2024-11-27T19:15:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:17,793 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:17,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631}] 2024-11-27T19:15:17,794 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:17,821 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 4ed13a3963bd22f54d2cd06886c65e56, had cached 0 bytes from a total of 28925 2024-11-27T19:15:17,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2024-11-27T19:15:17,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:17,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 44c0c476649e6e06a321127ef8d18631: 2024-11-27T19:15:17,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. for snapshot_1732734917373_default_test-1732734861815 completed. 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(241): Storing 'test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631.' region-info for snapshot=snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2] hfiles 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 for snapshot=snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 for snapshot=snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3] hfiles 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (1/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf for snapshot=snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:17,947 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(265): Adding reference for file (2/2): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 for snapshot=snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:17,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741925_1101 (size=224) 2024-11-27T19:15:18,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:18,296 INFO [master/eb850308c015:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T19:15:18,296 INFO [master/eb850308c015:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T19:15:18,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:18,353 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T19:15:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T19:15:18,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734917373_default_test-1732734861815 on region 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:18,354 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, ppid=56, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:18,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T19:15:18,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 44c0c476649e6e06a321127ef8d18631 in 562 msec 2024-11-27T19:15:18,356 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:18,357 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:18,358 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-27T19:15:18,358 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-27T19:15:18,358 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:18,359 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f 2024-11-27T19:15:18,359 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:18,359 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(328): No files under family: f2 2024-11-27T19:15:18,359 DEBUG [MobRegionSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:18,360 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631] hfiles 2024-11-27T19:15:18,360 DEBUG [MobRegionSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/1): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:18,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741926_1102 (size=168) 2024-11-27T19:15:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:18,767 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:18,767 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:18,768 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:18,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741927_1103 (size=1347) 2024-11-27T19:15:19,178 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:19,186 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:19,186 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734917373_default_test-1732734861815 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:19,187 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:19,187 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 }, snapshot procedure id = 56 2024-11-27T19:15:19,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1732734917373_default_test-1732734861815 table=test-1732734861815 type=FLUSH ttl=0 } in 1.8050 sec 2024-11-27T19:15:19,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T19:15:19,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: default:test-1732734861815 completed 2024-11-27T19:15:19,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } 2024-11-27T19:15:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734919534 (current time:1732734919534). 2024-11-27T19:15:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_1732734919533_ns4_test-17327348618153 VERSION not specified, setting to 2 2024-11-27T19:15:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318658b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:19,535 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:19,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:19,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:19,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41353e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:19,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:19,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:19,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:19,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:19,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b58f2ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:19,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:19,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:19,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:19,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:19,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } 2024-11-27T19:15:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-27T19:15:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:19,543 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:19,544 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:19,547 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:19,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741928_1104 (size=86) 2024-11-27T19:15:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:19,953 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:19,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5}] 2024-11-27T19:15:19,954 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:20,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f9ef0e14ba9a7626cfb7c31375818ae5: 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. for snapshot_1732734919533_ns4_test-17327348618153 completed. 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(241): Storing 'ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.' region-info for snapshot=snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:20,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-27T19:15:20,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741929_1105 (size=52) 2024-11-27T19:15:20,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:20,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:20,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-27T19:15:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-27T19:15:20,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_1732734919533_ns4_test-17327348618153 on region f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:20,513 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=59, ppid=58, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:20,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T19:15:20,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f9ef0e14ba9a7626cfb7c31375818ae5 in 561 msec 2024-11-27T19:15:20,515 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:20,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:20,516 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:20,517 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:20,517 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741930_1106 (size=443) 2024-11-27T19:15:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:20,927 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:20,932 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:20,932 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_1732734919533_ns4_test-17327348618153 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:20,934 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:20,934 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 }, snapshot procedure id = 58 2024-11-27T19:15:20,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_1732734919533_ns4_test-17327348618153 table=ns4:test-17327348618153 type=FLUSH ttl=0 } in 1.3930 sec 2024-11-27T19:15:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T19:15:21,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: ns4:test-17327348618153 completed 2024-11-27T19:15:21,704 DEBUG [Time-limited test {}] impl.FullTableBackupClient(174): snapshot copy for backup_1732734906555 2024-11-27T19:15:21,704 INFO [Time-limited test {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2024-11-27T19:15:21,704 DEBUG [Time-limited test {}] impl.FullTableBackupClient(80): There are 5 snapshots to be copied. 2024-11-27T19:15:21,704 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734906555_backup:system_bulk 2024-11-27T19:15:21,704 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734910896_backup_system_bulk to hdfs://localhost:44785/backupUT/backup_1732734906555/backup/system_bulk/ 2024-11-27T19:15:21,704 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:15:21,738 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:15:21,738 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734906555/backup/system_bulk, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734906555/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:21,740 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:15:21,743 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734910896_backup_system_bulk to hdfs://localhost:44785/backupUT/backup_1732734906555/backup/system_bulk/.hbase-snapshot/.tmp/snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741931_1107 (size=76) 2024-11-27T19:15:21,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741932_1108 (size=702) 2024-11-27T19:15:22,154 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:22,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:22,155 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,145 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-18069123807906614869.jar 2024-11-27T19:15:23,146 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,146 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,146 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,146 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,146 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,147 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,147 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,147 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,147 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:23,147 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:23,148 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:23,148 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:23,148 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:23,148 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:23,149 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:23,149 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:23,149 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:23,149 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:23,149 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:23,150 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:23,150 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:23,150 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:23,151 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:23,151 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:23,151 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:23,151 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:23,151 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:23,193 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:23,222 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734910896_backup_system_bulk' hfile list 2024-11-27T19:15:23,315 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1125273820_0005_ad0baa65-873c-40e6-bb2f-ebcd158ea3f4/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:23,315 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1125273820_0005_ad0baa65-873c-40e6-bb2f-ebcd158ea3f4/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:24,356 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:15:24,357 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:15:24,361 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:24,362 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734910896_backup_system_bulk finished. 2024-11-27T19:15:24,362 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734906555_ns2:test-17327348618151 2024-11-27T19:15:24,362 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734913053_ns2_test-17327348618151 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151/ 2024-11-27T19:15:24,362 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:15:24,392 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:15:24,392 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151/.hbase-snapshot/.tmp/snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:24,393 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:15:24,398 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734913053_ns2_test-17327348618151 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151/.hbase-snapshot/.tmp/snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:24,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741933_1109 (size=86) 2024-11-27T19:15:24,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741934_1110 (size=496) 2024-11-27T19:15:24,812 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:24,813 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:24,813 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,795 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-3430523832046335307.jar 2024-11-27T19:15:25,795 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,796 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:25,797 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:25,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:25,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:25,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:25,798 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:25,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:25,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:25,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:25,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:25,799 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:25,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:25,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:25,800 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:25,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:25,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:25,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:25,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:25,801 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:25,842 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:25,869 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734913053_ns2_test-17327348618151' hfile list 2024-11-27T19:15:25,871 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=13.2 K 2024-11-27T19:15:25,959 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1619507463_0006_e66a84cd-139a-4d82-a668-9f1712fdf476/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:25,959 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1619507463_0006_e66a84cd-139a-4d82-a668-9f1712fdf476/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:26,032 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-27T19:15:26,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741935_1111 (size=8413) 2024-11-27T19:15:26,476 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 output=hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151/archive/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/b15be9a8be414da1925a80e7914b3073 2024-11-27T19:15:26,477 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-27T19:15:26,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741936_1112 (size=5134) 2024-11-27T19:15:26,909 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b output=hdfs://localhost:44785/backupUT/backup_1732734906555/ns2/test-17327348618151/archive/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/f/aaa603a48f7c48c7910eb878894c8d3b 2024-11-27T19:15:26,909 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5134 (5.0 K) time=0sec 0.011M/sec 2024-11-27T19:15:27,001 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:15:27,002 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:15:27,008 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:27,008 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734913053_ns2_test-17327348618151 finished. 2024-11-27T19:15:27,008 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734906555_ns3:test-17327348618152 2024-11-27T19:15:27,008 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734915214_ns3_test-17327348618152 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns3/test-17327348618152/ 2024-11-27T19:15:27,008 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:15:27,039 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:15:27,039 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734906555/ns3/test-17327348618152, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734906555/ns3/test-17327348618152/.hbase-snapshot/.tmp/snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:27,041 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:15:27,045 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734915214_ns3_test-17327348618152 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns3/test-17327348618152/.hbase-snapshot/.tmp/snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741937_1113 (size=86) 2024-11-27T19:15:27,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741938_1114 (size=443) 2024-11-27T19:15:27,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:27,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:27,457 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,442 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-5485268633280327865.jar 2024-11-27T19:15:28,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,443 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,444 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:28,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:28,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:28,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:28,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:28,445 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:28,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:28,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:28,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:28,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:28,446 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:28,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:28,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:28,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:28,447 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:28,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:28,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:28,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:28,448 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:28,488 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:28,514 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734915214_ns3_test-17327348618152' hfile list 2024-11-27T19:15:28,604 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1189930818_0007_9da60946-6001-46f8-971b-6b248d95f4fe/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:28,604 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1189930818_0007_9da60946-6001-46f8-971b-6b248d95f4fe/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:29,644 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:15:29,645 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:15:29,650 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:29,650 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734915214_ns3_test-17327348618152 finished. 2024-11-27T19:15:29,650 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734906555_test-1732734861815 2024-11-27T19:15:29,650 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734917373_default_test-1732734861815 to hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/ 2024-11-27T19:15:29,650 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:15:29,679 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:15:29,679 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/.hbase-snapshot/.tmp/snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:29,680 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:15:29,686 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734917373_default_test-1732734861815 to hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/.hbase-snapshot/.tmp/snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:29,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741939_1115 (size=84) 2024-11-27T19:15:29,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741940_1116 (size=1347) 2024-11-27T19:15:30,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:30,100 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:30,101 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,084 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-3892865635929227812.jar 2024-11-27T19:15:31,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,085 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:31,086 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:31,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:31,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:31,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:31,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:31,087 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:31,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:31,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:31,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:31,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:31,088 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:31,089 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:31,089 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:31,089 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:31,089 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:31,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:31,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:31,090 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:31,130 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:31,156 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734917373_default_test-1732734861815' hfile list 2024-11-27T19:15:31,158 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(763): export split=0 size=43.0 K 2024-11-27T19:15:31,245 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1500844459_0008_10143a3c-3536-4a36-8f81-f394f4095380/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:31,245 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1500844459_0008_10143a3c-3536-4a36-8f81-f394f4095380/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:31,315 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(239): Using bufferSize=128 M 2024-11-27T19:15:31,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741941_1117 (size=13760) 2024-11-27T19:15:31,757 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 output=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/50f81c27462241ef93eff9714ab5e5c3 2024-11-27T19:15:31,757 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=13760 (13.4 K) time=0sec 0.030M/sec 2024-11-27T19:15:31,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741942_1118 (size=10796) 2024-11-27T19:15:32,190 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 output=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/0a20a56274a94f0d8fb0e7ccc4372ef2 2024-11-27T19:15:32,190 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=10796 (10.5 K) time=0sec 0.024M/sec 2024-11-27T19:15:32,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741943_1119 (size=8413) 2024-11-27T19:15:32,631 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 output=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f/ee663f7a5913448da72880447b51fed4 2024-11-27T19:15:32,632 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=8413 (8.2 K) time=0sec 0.018M/sec 2024-11-27T19:15:32,656 WARN [LocalJobRunner Map Task Executor #0 {}] mapred.Task(640): Task status: "copied 5.5 K/5.5 K (100.0%) from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 to hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631" truncated to max limit (512 characters) 2024-11-27T19:15:32,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741944_1120 (size=5591) 2024-11-27T19:15:33,064 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/mobdir/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 output=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/1bdf41c7fe9d0a2de8a8ff7d768527d1/mob/d41d8cd98f00b204e9800998ecf8427e202411275d8098d63a044a979def288975d4386a_44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:33,064 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5591 (5.5 K) time=0sec 0.012M/sec 2024-11-27T19:15:33,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741945_1121 (size=5484) 2024-11-27T19:15:33,504 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(340): copy completed for input=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf output=hdfs://localhost:44785/backupUT/backup_1732734906555/default/test-1732734861815/archive/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/mob/b947d32e22564efb834122adebb606cf 2024-11-27T19:15:33,504 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(342): size=5484 (5.4 K) time=0sec 0.012M/sec 2024-11-27T19:15:34,287 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:15:34,288 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:15:34,296 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:34,296 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734917373_default_test-1732734861815 finished. 2024-11-27T19:15:34,296 DEBUG [Time-limited test {}] impl.FullTableBackupClient(108): Setting snapshot copy job name to : Full-Backup_backup_1732734906555_ns4:test-17327348618153 2024-11-27T19:15:34,296 DEBUG [Time-limited test {}] impl.FullTableBackupClient(112): Copy snapshot snapshot_1732734919533_ns4_test-17327348618153 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns4/test-17327348618153/ 2024-11-27T19:15:34,296 DEBUG [Time-limited test {}] mapreduce.MapReduceBackupCopyJob(379): Doing SNAPSHOT_COPY 2024-11-27T19:15:34,325 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1094): inputFs=hdfs://localhost:44785, inputRoot=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9 2024-11-27T19:15:34,325 DEBUG [Time-limited test {}] snapshot.ExportSnapshot(1095): outputFs=DFS[DFSClient[clientName=DFSClient_NONMAPREDUCE_1467065364_20, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:44785/backupUT/backup_1732734906555/ns4/test-17327348618153, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:44785/backupUT/backup_1732734906555/ns4/test-17327348618153/.hbase-snapshot/.tmp/snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:34,327 INFO [Time-limited test {}] snapshot.ExportSnapshot(1104): Verify the source snapshot's expiration status and integrity. 2024-11-27T19:15:34,330 INFO [Time-limited test {}] snapshot.ExportSnapshot(1162): Copy Snapshot Manifest from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_1732734919533_ns4_test-17327348618153 to hdfs://localhost:44785/backupUT/backup_1732734906555/ns4/test-17327348618153/.hbase-snapshot/.tmp/snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:34,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741946_1122 (size=86) 2024-11-27T19:15:34,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741947_1123 (size=443) 2024-11-27T19:15:34,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:34,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:34,742 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,725 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-3037114027821818257.jar 2024-11-27T19:15:35,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,726 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,727 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:35,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:35,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:35,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:35,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:35,728 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:35,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:35,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:35,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:35,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:35,729 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:35,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:35,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:35,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:35,730 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:35,731 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:35,731 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-common/3.4.1/hadoop-common-3.4.1.jar 2024-11-27T19:15:35,731 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:35,731 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:35,770 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:35,796 INFO [Time-limited test {}] snapshot.ExportSnapshot(663): Loading Snapshot 'snapshot_1732734919533_ns4_test-17327348618153' hfile list 2024-11-27T19:15:35,884 WARN [Time-limited test {}] fs.FileUtil(1267): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1363938503_0009_5898330b-3b84-43fb-9458-9eff57dce1ac/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/*': No such file or directory 2024-11-27T19:15:35,884 WARN [Time-limited test {}] mapred.LocalDistributedCacheManager(200): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/mapreduce.cluster.local.dir/job_local1363938503_0009_5898330b-3b84-43fb-9458-9eff57dce1ac/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/libjars/* 2024-11-27T19:15:36,924 INFO [Time-limited test {}] snapshot.ExportSnapshot(1219): Finalize the Snapshot Export 2024-11-27T19:15:36,925 INFO [Time-limited test {}] snapshot.ExportSnapshot(1230): Verify the exported snapshot's expiration status and integrity. 2024-11-27T19:15:36,930 INFO [Time-limited test {}] snapshot.ExportSnapshot(1236): Export Completed: snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:36,930 INFO [Time-limited test {}] impl.FullTableBackupClient(124): Snapshot copy snapshot_1732734919533_ns4_test-17327348618153 finished. 2024-11-27T19:15:36,930 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): backup:system_bulk 2024-11-27T19:15:36,930 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns2:test-17327348618151 2024-11-27T19:15:36,930 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns3:test-17327348618152 2024-11-27T19:15:36,930 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): test-1732734861815 2024-11-27T19:15:36,930 DEBUG [Time-limited test {}] impl.BackupSystemTable(1041): ns4:test-17327348618153 2024-11-27T19:15:36,940 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734906555 2024-11-27T19:15:36,940 DEBUG [Time-limited test {}] impl.TableBackupClient(305): Current backup is a full backup, no direct ancestor for it. 2024-11-27T19:15:36,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741948_1124 (size=209) 2024-11-27T19:15:37,346 INFO [Time-limited test {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:44785/backupUT/backup_1732734906555/.backup.manifest 2024-11-27T19:15:37,346 DEBUG [Time-limited test {}] impl.TableBackupClient(387): Backup backup_1732734906555 finished: type=FULL,tablelist=backup:system_bulk;ns2:test-17327348618151;ns3:test-17327348618152;test-1732734861815;ns4:test-17327348618153,targetRootDir=hdfs://localhost:44785/backupUT,startts=1732734910763,completets=1732734936940,bytescopied=0 2024-11-27T19:15:37,346 DEBUG [Time-limited test {}] impl.TableBackupClient(142): Trying to delete snapshot for full backup. 2024-11-27T19:15:37,346 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:37,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734910896_backup_system_bulk" type: DISABLED 2024-11-27T19:15:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734910896_backup_system_bulk 2024-11-27T19:15:37,350 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734910896_backup_system_bulk for backup backup_1732734906555 succeeded. 2024-11-27T19:15:37,350 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:37,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734913053_ns2_test-17327348618151" type: DISABLED 2024-11-27T19:15:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734913053_ns2_test-17327348618151 2024-11-27T19:15:37,353 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734913053_ns2_test-17327348618151 for backup backup_1732734906555 succeeded. 2024-11-27T19:15:37,353 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:37,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734915214_ns3_test-17327348618152" type: DISABLED 2024-11-27T19:15:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734915214_ns3_test-17327348618152 2024-11-27T19:15:37,356 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734915214_ns3_test-17327348618152 for backup backup_1732734906555 succeeded. 2024-11-27T19:15:37,356 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:37,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734917373_default_test-1732734861815" type: DISABLED 2024-11-27T19:15:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734917373_default_test-1732734861815 2024-11-27T19:15:37,358 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734917373_default_test-1732734861815 for backup backup_1732734906555 succeeded. 2024-11-27T19:15:37,358 DEBUG [Time-limited test {}] impl.TableBackupClient(147): Trying to delete snapshot: snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:37,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_1732734919533_ns4_test-17327348618153" type: DISABLED 2024-11-27T19:15:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_1732734919533_ns4_test-17327348618153 2024-11-27T19:15:37,361 DEBUG [Time-limited test {}] impl.TableBackupClient(152): Deleting the snapshot snapshot_1732734919533_ns4_test-17327348618153 for backup backup_1732734906555 succeeded. 2024-11-27T19:15:37,362 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-27T19:15:37,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-27T19:15:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-27T19:15:37,367 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-27T19:15:37,369 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-27T19:15:37,374 INFO [Time-limited test {}] impl.TableBackupClient(405): Backup backup_1732734906555 completed. 2024-11-27T19:15:37,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:41,258 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T19:15:42,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,433 DEBUG [Time-limited test {}] impl.BackupSystemTable(635): Start new backup exclusive operation 2024-11-27T19:15:42,437 DEBUG [Time-limited test {}] impl.TableBackupClient(301): Getting the direct ancestors of the current backup backup_1732734942398 2024-11-27T19:15:42,439 DEBUG [Time-limited test {}] impl.TableBackupClient(334): Dependent full backup image: {BackupID=backup_1732734906555} 2024-11-27T19:15:42,439 DEBUG [Time-limited test {}] impl.TableBackupClient(337): Got 1 ancestors for the current backup. 2024-11-27T19:15:42,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:42,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734942524 (current time:1732734942524). 2024-11-27T19:15:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2024-11-27T19:15:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566337c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:42,526 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:42,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:42,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:42,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547889a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:42,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:42,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:42,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:42,528 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:42,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@487ef996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:42,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:42,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:42,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:42,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:42,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:42,532 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-27T19:15:42,535 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:42,536 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:42,538 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:42,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741949_1125 (size=52) 2024-11-27T19:15:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:42,944 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-27T19:15:42,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56}] 2024-11-27T19:15:42,946 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:43,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41349 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2024-11-27T19:15:43,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:43,098 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=2.54 KB heapSize=4.22 KB 2024-11-27T19:15:43,104 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/b5e99e29bc1d44a99508ba1d0aca28ce is 158, key is trslm:hdfs://localhost:44785/backupUT\x00ns2:test-17327348618151/meta:log-roll-map/1732734936935/Put/seqid=0 2024-11-27T19:15:43,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741950_1126 (size=6646) 2024-11-27T19:15:43,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:43,508 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/b5e99e29bc1d44a99508ba1d0aca28ce 2024-11-27T19:15:43,517 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/b327e11cdaec4fa19cc018160788e346 is 826, key is session:backup_1732734906555/session:context/1732734937369/Put/seqid=0 2024-11-27T19:15:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741951_1127 (size=5928) 2024-11-27T19:15:43,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:43,921 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.18 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/b327e11cdaec4fa19cc018160788e346 2024-11-27T19:15:43,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/b5e99e29bc1d44a99508ba1d0aca28ce as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce 2024-11-27T19:15:43,933 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce, entries=12, sequenceid=42, filesize=6.5 K 2024-11-27T19:15:43,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/b327e11cdaec4fa19cc018160788e346 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 2024-11-27T19:15:43,940 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346, entries=2, sequenceid=42, filesize=5.8 K 2024-11-27T19:15:43,941 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~2.54 KB/2604, heapSize ~4.19 KB/4288, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 843ms, sequenceid=42, compaction requested=true 2024-11-27T19:15:43,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:15:43,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. for snapshot_backup_system completed. 2024-11-27T19:15:43,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(241): Storing 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.' region-info for snapshot=snapshot_backup_system 2024-11-27T19:15:43,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-27T19:15:43,941 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce] hfiles 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/3): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346] hfiles 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (1/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (2/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (3/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee for snapshot=snapshot_backup_system 2024-11-27T19:15:43,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(265): Adding reference for file (4/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 for snapshot=snapshot_backup_system 2024-11-27T19:15:43,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741952_1128 (size=330) 2024-11-27T19:15:44,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:44,348 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-27T19:15:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-27T19:15:44,348 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.SnapshotRegionProcedure(132): finish snapshot snapshot_backup_system on region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:44,349 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=61, ppid=60, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:44,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T19:15:44,351 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-27T19:15:44,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56 in 1.4060 sec 2024-11-27T19:15:44,352 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:44,352 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:44,352 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for snapshot_backup_system 2024-11-27T19:15:44,353 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system 2024-11-27T19:15:44,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741953_1129 (size=967) 2024-11-27T19:15:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:44,762 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:44,770 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:44,771 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/snapshot_backup_system 2024-11-27T19:15:44,772 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:44,772 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2024-11-27T19:15:44,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2390 sec 2024-11-27T19:15:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T19:15:46,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-27T19:15:46,694 INFO [Time-limited test {}] impl.TableBackupClient(119): Backup backup_1732734942398 started at 1732734946694. 2024-11-27T19:15:46,696 DEBUG [Time-limited test {}] impl.TableBackupClient(123): Backup session backup_1732734942398 has been started. 2024-11-27T19:15:46,700 DEBUG [Time-limited test {}] impl.IncrementalTableBackupClient(285): For incremental backup, current table set is [backup:system_bulk, test-1732734861815, ns2:test-17327348618151, ns3:test-17327348618152, ns4:test-17327348618153] 2024-11-27T19:15:46,704 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(76): StartCode 1732734896959for backupID backup_1732734942398 2024-11-27T19:15:46,704 INFO [Time-limited test {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2024-11-27T19:15:46,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1035): Client=jenkins//172.17.0.2 procedure request for: rolllog-proc 2024-11-27T19:15:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure.ProcedureCoordinator(166): Submitting procedure rolllog 2024-11-27T19:15:46,705 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2024-11-27T19:15:46,705 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2024-11-27T19:15:46,705 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2024-11-27T19:15:46,705 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2024-11-27T19:15:46,705 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,705 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:46,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:15:46,710 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,710 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:15:46,710 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:15:46,710 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,710 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2024-11-27T19:15:46,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:46,711 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2024-11-27T19:15:46,711 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:46,711 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2024-11-27T19:15:46,712 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2024-11-27T19:15:46,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(141): Submitting new Subprocedure:rolllog 2024-11-27T19:15:46,712 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2024-11-27T19:15:46,712 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2024-11-27T19:15:46,712 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2024-11-27T19:15:46,712 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2024-11-27T19:15:46,712 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure (rolllog) in zk 2024-11-27T19:15:46,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,714 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:46,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:46,714 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,714 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2024-11-27T19:15:46,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:46,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:46,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:46,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:46,716 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'eb850308c015,41349,1732734853907' joining acquired barrier for procedure 'rolllog' on coordinator 2024-11-27T19:15:46,716 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@41113966[Count = 0] remaining members to acquire global barrier 2024-11-27T19:15:46,716 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2024-11-27T19:15:46,716 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,717 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,717 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,717 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2024-11-27T19:15:46,717 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,717 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2024-11-27T19:15:46,718 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2024-11-27T19:15:46,718 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: eb850308c015,41349,1732734853907 2024-11-27T19:15:46,718 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1732734910784 highest: 1732734910803 on eb850308c015,41349,1732734853907 2024-11-27T19:15:46,718 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0:(num 1732734910784) roll requested 2024-11-27T19:15:46,732 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734946718, exclude list is [], retry=0 2024-11-27T19:15:46,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:46,735 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 with entries=2, filesize=645 B; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734946718 2024-11-27T19:15:46,736 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:46,736 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 is not closed yet, will try archiving it next time 2024-11-27T19:15:46,736 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1:(num 1732734910803) roll requested 2024-11-27T19:15:46,736 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734878286 2024-11-27T19:15:46,737 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 not finished, retry = 0 2024-11-27T19:15:46,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741911_1087 (size=653) 2024-11-27T19:15:46,749 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734946736, exclude list is [], retry=0 2024-11-27T19:15:46,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:46,753 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803 with entries=17, filesize=4.59 KB; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734946736 2024-11-27T19:15:46,753 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:46,753 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803 is not closed yet, will try archiving it next time 2024-11-27T19:15:46,753 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL eb850308c015%2C41349%2C1732734853907.meta:.meta(num 1732734910821) roll requested 2024-11-27T19:15:46,753 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734896959 2024-11-27T19:15:46,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741912_1088 (size=4710) 2024-11-27T19:15:46,766 DEBUG [regionserver/eb850308c015:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734946753.meta, exclude list is [], retry=0 2024-11-27T19:15:46,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45013,DS-d5ee68fe-a069-4fae-bb51-774a35b0b379,DISK] 2024-11-27T19:15:46,769 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734946753.meta 2024-11-27T19:15:46,769 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33289:33289)] 2024-11-27T19:15:46,769 DEBUG [regionserver/eb850308c015:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta is not closed yet, will try archiving it next time 2024-11-27T19:15:46,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741913_1089 (size=93) 2024-11-27T19:15:46,770 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta not finished, retry = 0 2024-11-27T19:15:46,778 DEBUG [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2024-11-27T19:15:46,778 INFO [rs(eb850308c015,41349,1732734853907)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1732734946718 on eb850308c015,41349,1732734853907 2024-11-27T19:15:46,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system,, stopping at row=backup:system ,, for max=2147483647 with caching=100 2024-11-27T19:15:46,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:46,793 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2024-11-27T19:15:46,793 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'eb850308c015,41349,1732734853907' in zk 2024-11-27T19:15:46,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,795 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2024-11-27T19:15:46,795 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:15:46,795 DEBUG [member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2024-11-27T19:15:46,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:46,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:46,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:46,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:46,796 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:46,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:46,797 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:46,798 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'eb850308c015,41349,1732734853907': 2024-11-27T19:15:46,798 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2024-11-27T19:15:46,798 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'eb850308c015,41349,1732734853907' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2024-11-27T19:15:46,798 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2024-11-27T19:15:46,798 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2024-11-27T19:15:46,798 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2024-11-27T19:15:46,798 INFO [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllog including nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2024-11-27T19:15:46,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:15:46,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,801 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2024-11-27T19:15:46,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2024-11-27T19:15:46,801 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2024-11-27T19:15:46,802 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:15:46,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:15:46,802 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,802 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2024-11-27T19:15:46,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:46,803 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,803 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2024-11-27T19:15:46,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2024-11-27T19:15:46,804 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------eb850308c015,41349,1732734853907 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2024-11-27T19:15:46,806 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/eb850308c015,41349,1732734853907 2024-11-27T19:15:46,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2024-11-27T19:15:46,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2024-11-27T19:15:46,806 DEBUG [(eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2024-11-27T19:15:46,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2024-11-27T19:15:46,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2024-11-27T19:15:46,807 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2024-11-27T19:15:46,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2024-11-27T19:15:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1336): Checking to see if procedure from request:rolllog-proc is done 2024-11-27T19:15:46,815 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {eb850308c015:41349=1732734896959} newestTimestamps: {eb850308c015:41349=1732734910803} 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734857108.meta 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734896976.meta 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: eb850308c015%2C41349%2C1732734853907.meta.1732734946753.meta 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734946718 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803 2024-11-27T19:15:46,817 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734946736 2024-11-27T19:15:46,818 DEBUG [Time-limited test {}] impl.IncrementalBackupManager(213): Skip .meta log file: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.meta.1732734878325.meta 2024-11-27T19:15:46,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741957_1133 (size=640) 2024-11-27T19:15:46,838 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 2024-11-27T19:15:46,871 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs/eb850308c015%2C41349%2C1732734853907.meta.1732734910821.meta 2024-11-27T19:15:47,228 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:backup:system_bulk target: hdfs://localhost:44785/backupUT/backup_1732734942398/backup/system_bulk descriptor: 'backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:47,228 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:15:47,231 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table backup:system_bulk 2024-11-27T19:15:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741958_1134 (size=45) 2024-11-27T19:15:47,636 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table backup:system_bulk 2024-11-27T19:15:47,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741959_1135 (size=364) 2024-11-27T19:15:48,046 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns2:test-17327348618151 target: hdfs://localhost:44785/backupUT/backup_1732734942398/ns2/test-17327348618151 descriptor: 'ns2:test-17327348618151', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:48,046 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:15:48,049 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns2:test-17327348618151 2024-11-27T19:15:48,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741960_1136 (size=50) 2024-11-27T19:15:48,454 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns2:test-17327348618151 2024-11-27T19:15:48,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741961_1137 (size=390) 2024-11-27T19:15:48,865 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns3:test-17327348618152 target: hdfs://localhost:44785/backupUT/backup_1732734942398/ns3/test-17327348618152 descriptor: 'ns3:test-17327348618152', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:48,865 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:15:48,867 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns3:test-17327348618152 2024-11-27T19:15:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741962_1138 (size=50) 2024-11-27T19:15:49,272 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns3:test-17327348618152 2024-11-27T19:15:49,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741963_1139 (size=950) 2024-11-27T19:15:49,683 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:test-1732734861815 target: hdfs://localhost:44785/backupUT/backup_1732734942398/default/test-1732734861815 descriptor: 'test-1732734861815', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'mob', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '5', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:49,683 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:15:49,685 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table test-1732734861815 2024-11-27T19:15:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741964_1140 (size=53) 2024-11-27T19:15:50,090 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table test-1732734861815 2024-11-27T19:15:50,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741965_1141 (size=390) 2024-11-27T19:15:50,501 DEBUG [Time-limited test {}] util.BackupUtils(148): Attempting to copy table info for:ns4:test-17327348618153 target: hdfs://localhost:44785/backupUT/backup_1732734942398/ns4/test-17327348618153 descriptor: 'ns4:test-17327348618153', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T19:15:50,501 DEBUG [Time-limited test {}] util.BackupUtils(150): Finished copying tableinfo. 2024-11-27T19:15:50,503 DEBUG [Time-limited test {}] util.BackupUtils(153): Starting to write region info for table ns4:test-17327348618153 2024-11-27T19:15:50,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741966_1142 (size=50) 2024-11-27T19:15:50,908 DEBUG [Time-limited test {}] util.BackupUtils(160): Finished writing region info for table ns4:test-17327348618153 2024-11-27T19:15:50,918 DEBUG [Time-limited test {}] mapreduce.WALPlayer(319): add incremental job :hdfs://localhost:44785/backupUT/.tmp/backup_1732734942398 from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784;hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1.1732734910803 2024-11-27T19:15:50,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ce95c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:50,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:50,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:50,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43826c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:50,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:50,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355f4d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:50,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T19:15:50,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncConnectionImpl(321): The fetched master address is eb850308c015,39035,1732734852191 2024-11-27T19:15:50,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77655888 2024-11-27T19:15:50,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T19:15:50,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T19:15:50,927 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(636): bulkload locality sensitive enabled 2024-11-27T19:15:50,927 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table backup:system_bulk 2024-11-27T19:15:50,927 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=backup:system_bulk,, stopping at row=backup:system_bulk ,, for max=2147483647 with caching=100 2024-11-27T19:15:50,928 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:50,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:50,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:50,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:50,931 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for backup:system_bulk: backup:system_bulk; 2024-11-27T19:15:50,931 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table test-1732734861815 2024-11-27T19:15:50,931 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test-1732734861815,, stopping at row=test-1732734861815 ,, for max=2147483647 with caching=100 2024-11-27T19:15:50,933 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for test-1732734861815: test-1732734861815; 2024-11-27T19:15:50,933 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns2:test-17327348618151 2024-11-27T19:15:50,933 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns2:test-17327348618151,, stopping at row=ns2:test-17327348618151 ,, for max=2147483647 with caching=100 2024-11-27T19:15:50,935 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns2:test-17327348618151: ns2:test-17327348618151; 2024-11-27T19:15:50,935 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns3:test-17327348618152 2024-11-27T19:15:50,935 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns3:test-17327348618152,, stopping at row=ns3:test-17327348618152 ,, for max=2147483647 with caching=100 2024-11-27T19:15:50,936 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns3:test-17327348618152: ns3:test-17327348618152; 2024-11-27T19:15:50,936 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(502): Looking up current regions for table ns4:test-17327348618153 2024-11-27T19:15:50,936 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=ns4:test-17327348618153,, stopping at row=ns4:test-17327348618153 ,, for max=2147483647 with caching=100 2024-11-27T19:15:50,938 DEBUG [Time-limited test {}] mapreduce.HFileOutputFormat2(511): SplitPoint startkey for ns4:test-17327348618153: ns4:test-17327348618153; 2024-11-27T19:15:50,938 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(659): Configuring 5 reduce partitions to match current region count for all tables 2024-11-27T19:15:50,938 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(526): Writing partition information to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/hbase-staging/partitions_4fb755c0-36c9-48cb-871b-04bbdeae6a8f 2024-11-27T19:15:50,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741967_1143 (size=324) 2024-11-27T19:15:51,345 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:51,345 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-protocol-shaded/target/hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:51,345 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-client/target/hbase-client-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:51,404 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 44c0c476649e6e06a321127ef8d18631, had cached 0 bytes from a total of 38453 2024-11-27T19:15:52,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-523605900900770766.jar 2024-11-27T19:15:52,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,329 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-hadoop-compat/target/hbase-hadoop-compat-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics/target/hbase-metrics-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-metrics-api/target/hbase-metrics-api-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-replication/target/hbase-replication-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,330 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-http/target/hbase-http-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,331 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-procedure/target/hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,331 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-zookeeper/target/hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,331 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:52,331 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.9/hbase-shaded-gson-4.1.9.jar 2024-11-27T19:15:52,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.9/hbase-shaded-protobuf-4.1.9.jar 2024-11-27T19:15:52,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.9/hbase-shaded-netty-4.1.9.jar 2024-11-27T19:15:52,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.9/hbase-unsafe-4.1.9.jar 2024-11-27T19:15:52,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/zookeeper/zookeeper/3.8.4/zookeeper-3.8.4.jar 2024-11-27T19:15:52,332 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2024-11-27T19:15:52,333 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2024-11-27T19:15:52,333 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2024-11-27T19:15:52,333 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2024-11-27T19:15:52,333 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-common/target/hbase-common-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,334 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-mapreduce/target/hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar 2024-11-27T19:15:52,335 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/3.4.1/hadoop-mapreduce-client-core-3.4.1.jar 2024-11-27T19:15:52,335 INFO [Time-limited test {}] mapreduce.HFileOutputFormat2(679): Incremental output configured for tables: backup:system_bulk,default:test-1732734861815,ns2:test-17327348618151,ns3:test-17327348618152,ns4:test-17327348618153 2024-11-27T19:15:52,335 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:15:52,335 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.mapreduce.WALPlayer.createSubmittableJob(WALPlayer.java:342) at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:425) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:52,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:52,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:52,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:52,336 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.9/hbase-shaded-miscellaneous-4.1.9.jar 2024-11-27T19:15:53,319 DEBUG [Time-limited test {}] mapreduce.TableMapReduceUtil(972): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop-15305606808417635798.jar 2024-11-27T19:15:53,332 WARN [Time-limited test {}] mapreduce.JobResourceUploader(149): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2024-11-27T19:15:53,357 DEBUG [Time-limited test {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 for WAL files 2024-11-27T19:15:53,358 ERROR [Time-limited test {}] impl.TableBackupClient(232): Unexpected exception in incremental-backup: incremental copy backup_1732734942398File hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 does not exist. java.io.FileNotFoundException: File hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 does not exist. at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1294) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$DirListingIterator.(DistributedFileSystem.java:1268) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1213) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$25.doCall(DistributedFileSystem.java:1209) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.listLocatedStatus(DistributedFileSystem.java:1227) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem.listLocatedStatus(FileSystem.java:2263) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getFiles(WALInputFormat.java:356) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:321) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.mapreduce.WALInputFormat.getSplits(WALInputFormat.java:301) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:311) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:328) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:201) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1677) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job$11.run(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.submit(Job.java:1674) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1695) ~[hadoop-mapreduce-client-core-3.4.1.jar:?] at org.apache.hadoop.hbase.mapreduce.WALPlayer.run(WALPlayer.java:426) ~[hbase-mapreduce-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.walToHFiles(IncrementalTableBackupClient.java:426) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.convertWALsToHFiles(IncrementalTableBackupClient.java:398) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.IncrementalTableBackupClient.execute(IncrementalTableBackupClient.java:300) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:594) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:184) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T19:15:53,359 ERROR [Time-limited test {}] impl.TableBackupClient(244): BackupId=backup_1732734942398,startts=1732734946694,failedts=1732734953359,failedphase=PREPARE_INCREMENTAL,failedmessage=File hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/WALs/eb850308c015,41349,1732734853907/eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0.1732734910784 does not exist. 2024-11-27T19:15:53,359 DEBUG [Time-limited test {}] impl.BackupSystemTable(1537): Restoring backup:system from snapshot 2024-11-27T19:15:53,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable backup:system 2024-11-27T19:15:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=backup:system 2024-11-27T19:15:53,374 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734953374"}]},"ts":"1732734953374"} 2024-11-27T19:15:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T19:15:53,376 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLING in hbase:meta 2024-11-27T19:15:53,376 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set backup:system to state=DISABLING 2024-11-27T19:15:53,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=backup:system}] 2024-11-27T19:15:53,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, UNASSIGN}] 2024-11-27T19:15:53,383 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, UNASSIGN 2024-11-27T19:15:53,384 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=CLOSING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:53,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, UNASSIGN because future has completed 2024-11-27T19:15:53,386 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T19:15:53,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:15:53,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T19:15:53,538 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:53,539 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T19:15:53,539 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 4ed13a3963bd22f54d2cd06886c65e56, disabling compactions & flushes 2024-11-27T19:15:53,539 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:53,539 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:53,539 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. after waiting 0 ms 2024-11-27T19:15:53,539 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:53,539 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=398 B heapSize=1.04 KB 2024-11-27T19:15:53,545 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/644d36b36ddd4b938fea9d35f9c4c521 is 103, key is rslogts:hdfs://localhost:44785/backupUT\x00eb850308c015:41349/meta:rs-log-ts/1732734946792/Put/seqid=0 2024-11-27T19:15:53,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741968_1144 (size=5338) 2024-11-27T19:15:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T19:15:53,949 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/644d36b36ddd4b938fea9d35f9c4c521 2024-11-27T19:15:53,957 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/a6b79d718e1a46e888160073db55187e is 303, key is session:backup_1732734942398/session:context/1732734946695/Put/seqid=0 2024-11-27T19:15:53,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741969_1145 (size=5391) 2024-11-27T19:15:54,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T19:15:54,361 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=299 B at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/a6b79d718e1a46e888160073db55187e 2024-11-27T19:15:54,367 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/644d36b36ddd4b938fea9d35f9c4c521 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521 2024-11-27T19:15:54,372 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521, entries=1, sequenceid=47, filesize=5.2 K 2024-11-27T19:15:54,373 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/a6b79d718e1a46e888160073db55187e as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e 2024-11-27T19:15:54,377 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e, entries=1, sequenceid=47, filesize=5.3 K 2024-11-27T19:15:54,378 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 839ms, sequenceid=47, compaction requested=true 2024-11-27T19:15:54,382 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/recovered.edits/50.seqid, newMaxSeqId=50, maxSeqId=1 2024-11-27T19:15:54,383 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:54,383 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:54,383 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 4ed13a3963bd22f54d2cd06886c65e56: Waiting for close lock at 1732734953539Running coprocessor pre-close hooks at 1732734953539Disabling compacts and flushes for region at 1732734953539Disabling writes for close at 1732734953539Obtaining lock to block concurrent updates at 1732734953539Preparing flush snapshotting stores in 4ed13a3963bd22f54d2cd06886c65e56 at 1732734953539Finished memstore snapshotting backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., syncing WAL and waiting on mvcc, flushsize=dataSize=398, getHeapSize=1032, getOffHeapSize=0, getCellsCount=2 at 1732734953540 (+1 ms)Flushing stores of backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. at 1732734953540Flushing 4ed13a3963bd22f54d2cd06886c65e56/meta: creating writer at 1732734953540Flushing 4ed13a3963bd22f54d2cd06886c65e56/meta: appending metadata at 1732734953544 (+4 ms)Flushing 4ed13a3963bd22f54d2cd06886c65e56/meta: closing flushed file at 1732734953544Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: creating writer at 1732734953954 (+410 ms)Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: appending metadata at 1732734953957 (+3 ms)Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: closing flushed file at 1732734953957Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64be2873: reopening flushed file at 1732734954366 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32eb1ab6: reopening flushed file at 1732734954372 (+6 ms)Finished flush of dataSize ~398 B/398, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 839ms, sequenceid=47, compaction requested=true at 1732734954378 (+6 ms)Writing region close event to WAL at 1732734954379 (+1 ms)Running coprocessor post-close hooks at 1732734954383 (+4 ms)Closed at 1732734954383 2024-11-27T19:15:54,386 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:54,386 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=CLOSED 2024-11-27T19:15:54,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:15:54,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-27T19:15:54,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 in 1.0030 sec 2024-11-27T19:15:54,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-27T19:15:54,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, UNASSIGN in 1.0090 sec 2024-11-27T19:15:54,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T19:15:54,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=backup:system in 1.0160 sec 2024-11-27T19:15:54,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734954397"}]},"ts":"1732734954397"} 2024-11-27T19:15:54,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=DISABLED in hbase:meta 2024-11-27T19:15:54,399 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set backup:system to state=DISABLED 2024-11-27T19:15:54,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=backup:system in 1.0310 sec 2024-11-27T19:15:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T19:15:54,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: backup:system completed 2024-11-27T19:15:54,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.RawAsyncHBaseAdmin(2177): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:54,522 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1763): Client=jenkins//172.17.0.2 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1732734954522 (current time:1732734954522). 2024-11-27T19:15:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-27T19:15:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotDescriptionUtils(339): Snapshot hbase-failsafe-snapshot_backup_system-1732734954521 VERSION not specified, setting to 2 2024-11-27T19:15:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1acf0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ClusterIdFetcher(90): Going to request eb850308c015,39035,-1 for getting cluster id 2024-11-27T19:15:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T19:15:54,523 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f215ccd9-b3b1-442e-b841-1d7c367d5f1f' 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f215ccd9-b3b1-442e-b841-1d7c367d5f1f" 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a85bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb850308c015,39035,-1] 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T19:15:54,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:54,525 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T19:15:54,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d20288e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2024-11-27T19:15:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T19:15:54,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb850308c015,41349,1732734853907, seqNum=-1] 2024-11-27T19:15:54,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T19:15:54,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T19:15:54,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035. 2024-11-27T19:15:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.MasterRpcServices.snapshot(MasterRpcServices.java:1767) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T19:15:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:54,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(810): No existing snapshot, attempting snapshot... 2024-11-27T19:15:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1445): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-27T19:15:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:54,531 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-27T19:15:54,532 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-27T19:15:54,534 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-27T19:15:54,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741970_1146 (size=81) 2024-11-27T19:15:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:54,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 81ee8faa2e16b928d2ef19ee0c78f7f6, had cached 0 bytes from a total of 13547 2024-11-27T19:15:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:54,940 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2024-11-27T19:15:54,941 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(300): Storing region-info for snapshot. 2024-11-27T19:15:54,941 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(305): Creating references for hfiles 2024-11-27T19:15:54,941 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df] hfiles 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/4): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df 2024-11-27T19:15:54,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(366): Adding snapshot references for [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee] hfiles 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (1/5): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (2/5): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (3/5): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (4/5): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 2024-11-27T19:15:54,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(374): Adding reference for hfile (5/5): hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee 2024-11-27T19:15:54,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741971_1147 (size=409) 2024-11-27T19:15:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:55,350 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2024-11-27T19:15:55,351 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-27T19:15:55,352 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-27T19:15:55,352 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:55,353 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:55,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741972_1148 (size=1046) 2024-11-27T19:15:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:55,763 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-27T19:15:55,771 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-27T19:15:55,771 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1732734954521 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:55,772 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-27T19:15:55,773 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 66 2024-11-27T19:15:55,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1732734954521 table=backup:system type=FLUSH ttl=0 } in 1.2440 sec 2024-11-27T19:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T19:15:56,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: SNAPSHOT, Table Name: backup:system completed 2024-11-27T19:15:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732734942524 type: FLUSH version: 2 ttl: 0 ) 2024-11-27T19:15:56,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(1055): Restore snapshot=snapshot_backup_system as table=backup:system 2024-11-27T19:15:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T19:15:56,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741973_1149 (size=635) 2024-11-27T19:15:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T19:15:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T19:15:57,107 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2024-11-27T19:15:57,107 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2024-11-27T19:15:57,109 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(410): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2024-11-27T19:15:57,113 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(183): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732734942524 type: FLUSH version: 2 ttl: 0 2024-11-27T19:15:57,114 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(850): get table regions: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system 2024-11-27T19:15:57,116 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(862): found 1 regions for table=backup:system 2024-11-27T19:15:57,116 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(229): region to restore: 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,120 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:57,123 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/644d36b36ddd4b938fea9d35f9c4c521 2024-11-27T19:15:57,124 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:57,126 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/a6b79d718e1a46e888160073db55187e 2024-11-27T19:15:57,127 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(272): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732734942524 type: FLUSH version: 2 ttl: 0 2024-11-27T19:15:57,128 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732734957128"}]},"ts":"9223372036854775807"} 2024-11-27T19:15:57,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T19:15:57,131 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T19:15:57,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1732734957129"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732734957129"}]},"ts":"1732734957129"} 2024-11-27T19:15:57,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T19:15:57,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(585): Overwritten 1 regions to Meta 2024-11-27T19:15:57,133 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(586): Overwritten regions: [{ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T19:15:57,134 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(481): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2024-11-27T19:15:57,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1732734942524 type: FLUSH version: 2 ttl: 0 ) in 448 msec 2024-11-27T19:15:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-27T19:15:57,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin(2211): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:57,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "hbase-failsafe-snapshot_backup_system-1732734954521" type: DISABLED 2024-11-27T19:15:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1732734954521 2024-11-27T19:15:57,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.HMaster$12(2787): Client=jenkins//172.17.0.2 enable backup:system 2024-11-27T19:15:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:ENABLE_TABLE_PREPARE, hasLock=false; EnableTableProcedure table=backup:system 2024-11-27T19:15:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T19:15:57,337 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2024-11-27T19:15:57,338 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734957337"}]},"ts":"1732734957337"} 2024-11-27T19:15:57,339 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLING in hbase:meta 2024-11-27T19:15:57,340 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2024-11-27T19:15:57,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN}] 2024-11-27T19:15:57,341 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN 2024-11-27T19:15:57,342 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2024-11-27T19:15:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T19:15:57,492 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=OPENING, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:57,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN because future has completed 2024-11-27T19:15:57,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907}] 2024-11-27T19:15:57,649 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(132): Open backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:57,650 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7752): Opening region: {ENCODED => 4ed13a3963bd22f54d2cd06886c65e56, NAME => 'backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.', STARTKEY => '', ENDKEY => ''} 2024-11-27T19:15:57,650 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2024-11-27T19:15:57,650 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,650 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(898): Instantiated backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T19:15:57,650 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7794): checking encryption for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,650 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7797): checking classloading for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,652 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,652 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed13a3963bd22f54d2cd06886c65e56 columnFamilyName meta 2024-11-27T19:15:57,652 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T19:15:57,658 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce 2024-11-27T19:15:57,662 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 2024-11-27T19:15:57,666 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df 2024-11-27T19:15:57,666 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(327): Store=4ed13a3963bd22f54d2cd06886c65e56/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:15:57,666 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,667 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ed13a3963bd22f54d2cd06886c65e56 columnFamilyName session 2024-11-27T19:15:57,667 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T19:15:57,672 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c 2024-11-27T19:15:57,675 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 2024-11-27T19:15:57,679 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 2024-11-27T19:15:57,682 DEBUG [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee 2024-11-27T19:15:57,682 INFO [StoreOpener-4ed13a3963bd22f54d2cd06886c65e56-1 {}] regionserver.HStore(327): Store=4ed13a3963bd22f54d2cd06886c65e56/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T19:15:57,682 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1038): replaying wal for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,683 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,684 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,685 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1048): stopping wal replay for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,685 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1060): Cleaning up temporary data for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,685 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-27T19:15:57,686 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1093): writing seq id for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,687 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1114): Opened 4ed13a3963bd22f54d2cd06886c65e56; next sequenceid=51; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9844363040, jitterRate=-0.08317224681377411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-27T19:15:57,687 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:57,688 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1006): Region open journal for 4ed13a3963bd22f54d2cd06886c65e56: Running coprocessor pre-open hook at 1732734957650Writing region info on filesystem at 1732734957650Initializing all the Stores at 1732734957651 (+1 ms)Instantiating store for column family {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734957651Instantiating store for column family {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732734957651Cleaning up temporary data from old regions at 1732734957685 (+34 ms)Running coprocessor post-open hooks at 1732734957687 (+2 ms)Region opened successfully at 1732734957687 2024-11-27T19:15:57,688 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2236): Post open deploy tasks for backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., pid=70, masterSystemTime=1732734957646 2024-11-27T19:15:57,690 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 4ed13a3963bd22f54d2cd06886c65e56:meta, priority=-2147483648, current under compaction store size is 1 2024-11-27T19:15:57,691 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T19:15:57,691 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(403): Add compact mark for store 4ed13a3963bd22f54d2cd06886c65e56:session, priority=-2147483648, current under compaction store size is 2 2024-11-27T19:15:57,692 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T19:15:57,692 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T19:15:57,692 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T19:15:57,693 DEBUG [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2266): Finished post open deploy task for backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:57,693 INFO [RS_OPEN_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(153): Opened backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:57,694 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T19:15:57,694 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 23194 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T19:15:57,694 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=4ed13a3963bd22f54d2cd06886c65e56, regionState=OPEN, openSeqNum=51, regionLocation=eb850308c015,41349,1732734853907 2024-11-27T19:15:57,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 because future has completed 2024-11-27T19:15:57,696 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HStore(1541): 4ed13a3963bd22f54d2cd06886c65e56/meta is initiating minor compaction (all files) 2024-11-27T19:15:57,696 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HStore(1541): 4ed13a3963bd22f54d2cd06886c65e56/session is initiating minor compaction (all files) 2024-11-27T19:15:57,696 INFO [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4ed13a3963bd22f54d2cd06886c65e56/meta in backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:57,696 INFO [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4ed13a3963bd22f54d2cd06886c65e56/session in backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:57,696 INFO [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce] into tmpdir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp, totalSize=17.9 K 2024-11-27T19:15:57,696 INFO [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346] into tmpdir=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp, totalSize=22.7 K 2024-11-27T19:15:57,697 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.Compactor(225): Compacting df512fd414b94e848ba4beffec983c91, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732734876051 2024-11-27T19:15:57,697 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec5d3c0a5f74c6685cb6ca50abbf0d9, keycount=6, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732734878371 2024-11-27T19:15:57,698 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b2e5a100664b3cac810fb2b372a8df, keycount=4, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732734897017 2024-11-27T19:15:57,698 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.Compactor(225): Compacting 22887eff9aa5454db46c9e02a88f862c, keycount=2, bloomtype=ROW, size=5.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732734890890 2024-11-27T19:15:57,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-27T19:15:57,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 4ed13a3963bd22f54d2cd06886c65e56, server=eb850308c015,41349,1732734853907 in 201 msec 2024-11-27T19:15:57,698 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e99e29bc1d44a99508ba1d0aca28ce, keycount=12, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732734910863 2024-11-27T19:15:57,698 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.Compactor(225): Compacting e59036f545b9458eadcf9f12163ddeee, keycount=3, bloomtype=ROW, size=6.5 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732734905478 2024-11-27T19:15:57,699 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] compactions.Compactor(225): Compacting b327e11cdaec4fa19cc018160788e346, keycount=2, bloomtype=ROW, size=5.8 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732734937369 2024-11-27T19:15:57,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T19:15:57,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=backup:system, region=4ed13a3963bd22f54d2cd06886c65e56, ASSIGN in 358 msec 2024-11-27T19:15:57,704 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732734957704"}]},"ts":"1732734957704"} 2024-11-27T19:15:57,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=backup:system, state=ENABLED in hbase:meta 2024-11-27T19:15:57,706 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2024-11-27T19:15:57,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; EnableTableProcedure table=backup:system in 375 msec 2024-11-27T19:15:57,725 INFO [RS:0;eb850308c015:41349-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4ed13a3963bd22f54d2cd06886c65e56#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T19:15:57,725 INFO [RS:0;eb850308c015:41349-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4ed13a3963bd22f54d2cd06886c65e56#meta#compaction#16 average throughput is 1.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T19:15:57,726 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/31c1fe4a193f4b95a4e194c1a6230a3a is 158, key is trslm:hdfs://localhost:44785/backupUT\x00ns2:test-17327348618151/meta:log-roll-map/1732734936935/Put/seqid=0 2024-11-27T19:15:57,726 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/46860a6c0f4b4caeb1ad77026420fe93 is 1245, key is session:backup_1732734906466/session:context/1732734906551/Put/seqid=0 2024-11-27T19:15:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741975_1151 (size=8073) 2024-11-27T19:15:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741974_1150 (size=6749) 2024-11-27T19:15:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T19:15:57,964 DEBUG [Time-limited test {}] impl.BackupSystemTable(1544): Done restoring backup system table 2024-11-27T19:15:57,964 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: ENABLE, Table Name: backup:system completed 2024-11-27T19:15:57,964 DEBUG [Time-limited test {}] impl.BackupSystemTable(1571): Deleting snapshot_backup_system from the system 2024-11-27T19:15:57,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] master.MasterRpcServices(838): Client=jenkins//172.17.0.2 delete name: "snapshot_backup_system" type: DISABLED 2024-11-27T19:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39035 {}] snapshot.SnapshotManager(381): Deleting snapshot: snapshot_backup_system 2024-11-27T19:15:57,970 DEBUG [Time-limited test {}] impl.BackupSystemTable(1576): Done deleting backup system table snapshot 2024-11-27T19:15:57,970 DEBUG [Time-limited test {}] impl.TableBackupClient(188): Trying to cleanup up target dir. Current backup phase: PREPARE_INCREMENTAL 2024-11-27T19:15:57,972 DEBUG [Time-limited test {}] impl.BackupSystemTable(662): Finish backup exclusive operation 2024-11-27T19:15:57,977 ERROR [Time-limited test {}] impl.TableBackupClient(250): Backup backup_1732734942398 failed. 2024-11-27T19:15:57,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:15:57,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.TestIncrementalBackup.TestIncBackupRestore(TestIncrementalBackup.java:103) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.runners.ParentRunner.run(ParentRunner.java:413) at org.junit.runners.Suite.runChild(Suite.java:128) at org.junit.runners.Suite.runChild(Suite.java:27) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:57,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:57,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:57,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:57,999 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackup#TestIncBackupRestore[0] Thread=627 (was 611) Potentially hanging thread: Timer for 'JobTracker' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-496223468-172.17.0.2-1732734849103:blk_1073741954_1130, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: WAL-Archive-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;eb850308c015:41349-shortCompactions-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:1013) app//org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:949) app//org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:913) app//org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:861) app//org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:77) app//org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:859) app//org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:680) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter$SingleStoreFileWriter.close(StoreFileWriter.java:787) app//org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:294) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:70) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:38) app//org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:398) app//org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:62) app//org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) app//org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1181) app//org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2424) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) app//org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-496223468-172.17.0.2-1732734849103:blk_1073741955_1131, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/eb850308c015:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(eb850308c015,41349,1732734853907)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(eb850308c015,41349,1732734853907)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2083293204_20 at /127.0.0.1:60158 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: (eb850308c015,39035,1732734852191)-proc-coordinator-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/46860a6c0f4b4caeb1ad77026420fe93 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: PacketResponder: BP-496223468-172.17.0.2-1732734849103:blk_1073741956_1132, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1559257797_20 at /127.0.0.1:58076 [Receiving block BP-496223468-172.17.0.2-1732734849103:blk_1073741954_1130] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(eb850308c015,41349,1732734853907)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/eb850308c015:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1559257797_20 at /127.0.0.1:58090 [Receiving block BP-496223468-172.17.0.2-1732734849103:blk_1073741955_1131] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1559257797_20 at /127.0.0.1:58098 [Receiving block BP-496223468-172.17.0.2-1732734849103:blk_1073741956_1132] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/31c1fe4a193f4b95a4e194c1a6230a3a java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:717) Potentially hanging thread: member: 'eb850308c015,41349,1732734853907' subprocedure-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack$SNode.block(SynchronousQueue.java:288) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:397) java.base@17.0.11/java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:886) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: rs(eb850308c015,41349,1732734853907)-backup-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=639 (was 633) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=124 (was 137), ProcessCount=15 (was 15), AvailableMemoryMB=5148 (was 5721) 2024-11-27T19:15:58,000 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=627 is superior to 500 2024-11-27T19:15:58,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T19:15:58,006 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:15:58,006 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:58,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:58,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:58,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:58,007 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T19:15:58,007 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=224113150, stopped=false 2024-11-27T19:15:58,007 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eb850308c015,39035,1732734852191 2024-11-27T19:15:58,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-27T19:15:58,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/running 2024-11-27T19:15:58,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:15:58,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:15:58,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T19:15:58,009 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T19:15:58,009 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-27T19:15:58,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.backup.TestBackupBase.tearDown(TestBackupBase.java:378) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:58,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:58,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2024-11-27T19:15:58,010 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eb850308c015,41349,1732734853907' ***** 2024-11-27T19:15:58,010 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T19:15:58,010 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-27T19:15:58,014 INFO [RS:0;eb850308c015:41349 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@9984275{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/regionserver} 2024-11-27T19:15:58,016 INFO [RS:0;eb850308c015:41349 {}] server.AbstractConnector(383): Stopped ServerConnector@445a8045{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-27T19:15:58,016 INFO [RS:0;eb850308c015:41349 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:15:58,016 INFO [RS:0;eb850308c015:41349 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2df3a35c{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-27T19:15:58,017 INFO [RS:0;eb850308c015:41349 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3e9615b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:15:58,018 INFO [RS:0;eb850308c015:41349 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T19:15:58,018 INFO [RS:0;eb850308c015:41349 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T19:15:58,018 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T19:15:58,018 INFO [RS:0;eb850308c015:41349 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for 4ed13a3963bd22f54d2cd06886c65e56 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for 44c0c476649e6e06a321127ef8d18631 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for 81ee8faa2e16b928d2ef19ee0c78f7f6 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for 05e9d065d2547a02238ecbaeb2f3acbc 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for cf00efa8e14baf8163e951bfbb0a60c6 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(959): stopping server eb850308c015,41349,1732734853907 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T19:15:58,019 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4ed13a3963bd22f54d2cd06886c65e56, disabling compactions & flushes 2024-11-27T19:15:58,019 INFO [RS:0;eb850308c015:41349 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eb850308c015:41349. 2024-11-27T19:15:58,020 DEBUG [RS:0;eb850308c015:41349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:58,020 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,020 DEBUG [RS:0;eb850308c015:41349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:58,020 INFO [RS:0;eb850308c015:41349 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T19:15:58,051 INFO [regionserver/eb850308c015:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T19:15:58,139 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/46860a6c0f4b4caeb1ad77026420fe93 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/46860a6c0f4b4caeb1ad77026420fe93 2024-11-27T19:15:58,139 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/meta/31c1fe4a193f4b95a4e194c1a6230a3a as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/31c1fe4a193f4b95a4e194c1a6230a3a 2024-11-27T19:15:58,152 INFO [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4ed13a3963bd22f54d2cd06886c65e56/meta of 4ed13a3963bd22f54d2cd06886c65e56 into 31c1fe4a193f4b95a4e194c1a6230a3a(size=6.6 K), total size for store is 6.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T19:15:58,152 INFO [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4ed13a3963bd22f54d2cd06886c65e56/session of 4ed13a3963bd22f54d2cd06886c65e56 into 46860a6c0f4b4caeb1ad77026420fe93(size=7.9 K), total size for store is 7.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T19:15:58,152 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:15:58,152 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4ed13a3963bd22f54d2cd06886c65e56: 2024-11-27T19:15:58,152 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,152 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,152 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. after waiting 0 ms 2024-11-27T19:15:58,152 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,152 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4ed13a3963bd22f54d2cd06886c65e56 2/2 column families, dataSize=594 B heapSize=1.23 KB 2024-11-27T19:15:58,154 INFO [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., storeName=4ed13a3963bd22f54d2cd06886c65e56/meta, priority=13, startTime=1732734957690; duration=0sec 2024-11-27T19:15:58,154 INFO [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., storeName=4ed13a3963bd22f54d2cd06886c65e56/session, priority=12, startTime=1732734957691; duration=0sec 2024-11-27T19:15:58,154 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T19:15:58,154 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T19:15:58,154 DEBUG [RS:0;eb850308c015:41349-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4ed13a3963bd22f54d2cd06886c65e56:meta 2024-11-27T19:15:58,154 DEBUG [RS:0;eb850308c015:41349-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4ed13a3963bd22f54d2cd06886c65e56:session 2024-11-27T19:15:58,155 INFO [RS:0;eb850308c015:41349 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T19:15:58,155 INFO [RS:0;eb850308c015:41349 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T19:15:58,155 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T19:15:58,155 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-27T19:15:58,155 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T19:15:58,155 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1325): Online Regions={4ed13a3963bd22f54d2cd06886c65e56=backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., 44c0c476649e6e06a321127ef8d18631=test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., 1588230740=hbase:meta,,1.1588230740, 81ee8faa2e16b928d2ef19ee0c78f7f6=ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6., 05e9d065d2547a02238ecbaeb2f3acbc=backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc., cf00efa8e14baf8163e951bfbb0a60c6=ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6., f9ef0e14ba9a7626cfb7c31375818ae5=ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5.} 2024-11-27T19:15:58,155 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T19:15:58,155 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T19:15:58,155 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T19:15:58,155 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T19:15:58,156 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.17 KB heapSize=23.40 KB 2024-11-27T19:15:58,156 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 05e9d065d2547a02238ecbaeb2f3acbc, 1588230740, 44c0c476649e6e06a321127ef8d18631, 4ed13a3963bd22f54d2cd06886c65e56, 81ee8faa2e16b928d2ef19ee0c78f7f6, cf00efa8e14baf8163e951bfbb0a60c6, f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:58,157 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/6a8a43e843374994b9ed26d86a96d9bb is 554, key is session:backup_1732734942398/session:context/1732734957971/Put/seqid=0 2024-11-27T19:15:58,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741976_1152 (size=5655) 2024-11-27T19:15:58,178 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/info/44a5e2682714423889309abcfe239c50 is 159, key is ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6./info:regioninfo/1732734864784/Put/seqid=0 2024-11-27T19:15:58,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741977_1153 (size=14126) 2024-11-27T19:15:58,356 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 05e9d065d2547a02238ecbaeb2f3acbc, 1588230740, 44c0c476649e6e06a321127ef8d18631, 4ed13a3963bd22f54d2cd06886c65e56, 81ee8faa2e16b928d2ef19ee0c78f7f6, cf00efa8e14baf8163e951bfbb0a60c6, f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:58,472 INFO [regionserver/eb850308c015:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T19:15:58,472 INFO [regionserver/eb850308c015:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T19:15:58,556 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 05e9d065d2547a02238ecbaeb2f3acbc, 1588230740, 44c0c476649e6e06a321127ef8d18631, 4ed13a3963bd22f54d2cd06886c65e56, 81ee8faa2e16b928d2ef19ee0c78f7f6, cf00efa8e14baf8163e951bfbb0a60c6, f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:58,562 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=594 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/6a8a43e843374994b9ed26d86a96d9bb 2024-11-27T19:15:58,568 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/.tmp/session/6a8a43e843374994b9ed26d86a96d9bb as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/6a8a43e843374994b9ed26d86a96d9bb 2024-11-27T19:15:58,573 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/6a8a43e843374994b9ed26d86a96d9bb, entries=2, sequenceid=57, filesize=5.5 K 2024-11-27T19:15:58,574 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 422ms, sequenceid=57, compaction requested=false 2024-11-27T19:15:58,574 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce] to archive 2024-11-27T19:15:58,575 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T19:15:58,578 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/cec5d3c0a5f74c6685cb6ca50abbf0d9 2024-11-27T19:15:58,579 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/d3b2e5a100664b3cac810fb2b372a8df 2024-11-27T19:15:58,580 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/meta/b5e99e29bc1d44a99508ba1d0aca28ce 2024-11-27T19:15:58,583 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.16 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/info/44a5e2682714423889309abcfe239c50 2024-11-27T19:15:58,586 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=eb850308c015:39035 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-27T19:15:58,586 WARN [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [cec5d3c0a5f74c6685cb6ca50abbf0d9=5914, d3b2e5a100664b3cac810fb2b372a8df=5745, b5e99e29bc1d44a99508ba1d0aca28ce=6646] 2024-11-27T19:15:58,587 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346] to archive 2024-11-27T19:15:58,587 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T19:15:58,590 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/df512fd414b94e848ba4beffec983c91 2024-11-27T19:15:58,591 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/22887eff9aa5454db46c9e02a88f862c 2024-11-27T19:15:58,592 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/e59036f545b9458eadcf9f12163ddeee 2024-11-27T19:15:58,593 DEBUG [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 to hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/archive/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/session/b327e11cdaec4fa19cc018160788e346 2024-11-27T19:15:58,593 WARN [StoreCloser-backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [df512fd414b94e848ba4beffec983c91=5049, 22887eff9aa5454db46c9e02a88f862c=5535, e59036f545b9458eadcf9f12163ddeee=6682, b327e11cdaec4fa19cc018160788e346=5928] 2024-11-27T19:15:58,597 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system/4ed13a3963bd22f54d2cd06886c65e56/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=50 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:58,598 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4ed13a3963bd22f54d2cd06886c65e56: Waiting for close lock at 1732734958019Running coprocessor pre-close hooks at 1732734958019Disabling compacts and flushes for region at 1732734958019Disabling writes for close at 1732734958152 (+133 ms)Obtaining lock to block concurrent updates at 1732734958152Preparing flush snapshotting stores in 4ed13a3963bd22f54d2cd06886c65e56 at 1732734958152Finished memstore snapshotting backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56., syncing WAL and waiting on mvcc, flushsize=dataSize=594, getHeapSize=1224, getOffHeapSize=0, getCellsCount=2 at 1732734958153 (+1 ms)Flushing stores of backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. at 1732734958153Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: creating writer at 1732734958153Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: appending metadata at 1732734958157 (+4 ms)Flushing 4ed13a3963bd22f54d2cd06886c65e56/session: closing flushed file at 1732734958157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24aeb60a: reopening flushed file at 1732734958568 (+411 ms)Finished flush of dataSize ~594 B/594, heapSize ~984 B/984, currentSize=0 B/0 for 4ed13a3963bd22f54d2cd06886c65e56 in 422ms, sequenceid=57, compaction requested=false at 1732734958574 (+6 ms)Writing region close event to WAL at 1732734958594 (+20 ms)Running coprocessor post-close hooks at 1732734958598 (+4 ms)Closed at 1732734958598 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1732734871670.4ed13a3963bd22f54d2cd06886c65e56. 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44c0c476649e6e06a321127ef8d18631, disabling compactions & flushes 2024-11-27T19:15:58,598 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. after waiting 0 ms 2024-11-27T19:15:58,598 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:58,599 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 44c0c476649e6e06a321127ef8d18631 3/3 column families, dataSize=378 B heapSize=1.63 KB 2024-11-27T19:15:58,604 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/ns/a7e0a8b8d93845e996bcfd88d586df72 is 43, key is default/ns:d/1732734857268/Put/seqid=0 2024-11-27T19:15:58,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741978_1154 (size=5345) 2024-11-27T19:15:58,615 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f2/a1443b52ce5e476db836a586b8407cbf is 58, key is row-test-1732734861815-2-0/f2:q1/1732734937390/Put/seqid=0 2024-11-27T19:15:58,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741979_1155 (size=5464) 2024-11-27T19:15:58,756 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 05e9d065d2547a02238ecbaeb2f3acbc, 1588230740, 44c0c476649e6e06a321127ef8d18631, 81ee8faa2e16b928d2ef19ee0c78f7f6, cf00efa8e14baf8163e951bfbb0a60c6, f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:58,957 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 05e9d065d2547a02238ecbaeb2f3acbc, 1588230740, 44c0c476649e6e06a321127ef8d18631, 81ee8faa2e16b928d2ef19ee0c78f7f6, cf00efa8e14baf8163e951bfbb0a60c6, f9ef0e14ba9a7626cfb7c31375818ae5 2024-11-27T19:15:59,008 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=235 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/ns/a7e0a8b8d93845e996bcfd88d586df72 2024-11-27T19:15:59,019 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=378 B at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f2/a1443b52ce5e476db836a586b8407cbf 2024-11-27T19:15:59,025 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/.tmp/f2/a1443b52ce5e476db836a586b8407cbf as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f2/a1443b52ce5e476db836a586b8407cbf 2024-11-27T19:15:59,030 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/f2/a1443b52ce5e476db836a586b8407cbf, entries=7, sequenceid=348, filesize=5.3 K 2024-11-27T19:15:59,032 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 433ms, sequenceid=348, compaction requested=false 2024-11-27T19:15:59,032 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/table/db1df5bc1ca840d48e6c041e3b5aadc7 is 59, key is ns2:test-17327348618151/table:state/1732734864797/Put/seqid=0 2024-11-27T19:15:59,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741980_1156 (size=5866) 2024-11-27T19:15:59,036 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/default/test-1732734861815/44c0c476649e6e06a321127ef8d18631/recovered.edits/351.seqid, newMaxSeqId=351, maxSeqId=337 2024-11-27T19:15:59,037 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,037 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:59,037 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44c0c476649e6e06a321127ef8d18631: Waiting for close lock at 1732734958598Running coprocessor pre-close hooks at 1732734958598Disabling compacts and flushes for region at 1732734958598Disabling writes for close at 1732734958598Obtaining lock to block concurrent updates at 1732734958599 (+1 ms)Preparing flush snapshotting stores in 44c0c476649e6e06a321127ef8d18631 at 1732734958599Finished memstore snapshotting test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631., syncing WAL and waiting on mvcc, flushsize=dataSize=378, getHeapSize=1616, getOffHeapSize=0, getCellsCount=7 at 1732734958599Flushing stores of test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. at 1732734958599Flushing 44c0c476649e6e06a321127ef8d18631/f2: creating writer at 1732734958599Flushing 44c0c476649e6e06a321127ef8d18631/f2: appending metadata at 1732734958614 (+15 ms)Flushing 44c0c476649e6e06a321127ef8d18631/f2: closing flushed file at 1732734958614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd3d90: reopening flushed file at 1732734959024 (+410 ms)Finished flush of dataSize ~378 B/378, heapSize ~1.11 KB/1136, currentSize=0 B/0 for 44c0c476649e6e06a321127ef8d18631 in 433ms, sequenceid=348, compaction requested=false at 1732734959032 (+8 ms)Writing region close event to WAL at 1732734959033 (+1 ms)Running coprocessor post-close hooks at 1732734959037 (+4 ms)Closed at 1732734959037 2024-11-27T19:15:59,037 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1732734861815,,1732734862299.44c0c476649e6e06a321127ef8d18631. 2024-11-27T19:15:59,037 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 81ee8faa2e16b928d2ef19ee0c78f7f6, disabling compactions & flushes 2024-11-27T19:15:59,038 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:59,038 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:59,038 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. after waiting 0 ms 2024-11-27T19:15:59,038 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:59,041 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns2/test-17327348618151/81ee8faa2e16b928d2ef19ee0c78f7f6/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,042 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 81ee8faa2e16b928d2ef19ee0c78f7f6: Waiting for close lock at 1732734959037Running coprocessor pre-close hooks at 1732734959037Disabling compacts and flushes for region at 1732734959037Disabling writes for close at 1732734959038 (+1 ms)Writing region close event to WAL at 1732734959038Running coprocessor post-close hooks at 1732734959042 (+4 ms)Closed at 1732734959042 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17327348618151,,1732734863608.81ee8faa2e16b928d2ef19ee0c78f7f6. 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05e9d065d2547a02238ecbaeb2f3acbc, disabling compactions & flushes 2024-11-27T19:15:59,042 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. after waiting 0 ms 2024-11-27T19:15:59,042 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:59,045 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/backup/system_bulk/05e9d065d2547a02238ecbaeb2f3acbc/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,046 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05e9d065d2547a02238ecbaeb2f3acbc: Waiting for close lock at 1732734959042Running coprocessor pre-close hooks at 1732734959042Disabling compacts and flushes for region at 1732734959042Disabling writes for close at 1732734959042Writing region close event to WAL at 1732734959042Running coprocessor post-close hooks at 1732734959046 (+4 ms)Closed at 1732734959046 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1732734873853.05e9d065d2547a02238ecbaeb2f3acbc. 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf00efa8e14baf8163e951bfbb0a60c6, disabling compactions & flushes 2024-11-27T19:15:59,046 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. after waiting 0 ms 2024-11-27T19:15:59,046 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:59,049 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns3/test-17327348618152/cf00efa8e14baf8163e951bfbb0a60c6/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,050 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf00efa8e14baf8163e951bfbb0a60c6: Waiting for close lock at 1732734959046Running coprocessor pre-close hooks at 1732734959046Disabling compacts and flushes for region at 1732734959046Disabling writes for close at 1732734959046Writing region close event to WAL at 1732734959046Running coprocessor post-close hooks at 1732734959050 (+4 ms)Closed at 1732734959050 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17327348618152,,1732734865924.cf00efa8e14baf8163e951bfbb0a60c6. 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9ef0e14ba9a7626cfb7c31375818ae5, disabling compactions & flushes 2024-11-27T19:15:59,050 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. after waiting 0 ms 2024-11-27T19:15:59,050 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:59,053 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/ns4/test-17327348618153/f9ef0e14ba9a7626cfb7c31375818ae5/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-27T19:15:59,053 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,053 INFO [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:59,053 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9ef0e14ba9a7626cfb7c31375818ae5: Waiting for close lock at 1732734959050Running coprocessor pre-close hooks at 1732734959050Disabling compacts and flushes for region at 1732734959050Disabling writes for close at 1732734959050Writing region close event to WAL at 1732734959050Running coprocessor post-close hooks at 1732734959053 (+3 ms)Closed at 1732734959053 2024-11-27T19:15:59,053 DEBUG [RS_CLOSE_REGION-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17327348618153,,1732734868084.f9ef0e14ba9a7626cfb7c31375818ae5. 2024-11-27T19:15:59,157 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T19:15:59,157 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T19:15:59,157 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T19:15:59,357 DEBUG [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T19:15:59,437 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/table/db1df5bc1ca840d48e6c041e3b5aadc7 2024-11-27T19:15:59,443 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/info/44a5e2682714423889309abcfe239c50 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/info/44a5e2682714423889309abcfe239c50 2024-11-27T19:15:59,447 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/info/44a5e2682714423889309abcfe239c50, entries=69, sequenceid=59, filesize=13.8 K 2024-11-27T19:15:59,448 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/ns/a7e0a8b8d93845e996bcfd88d586df72 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/ns/a7e0a8b8d93845e996bcfd88d586df72 2024-11-27T19:15:59,453 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/ns/a7e0a8b8d93845e996bcfd88d586df72, entries=7, sequenceid=59, filesize=5.2 K 2024-11-27T19:15:59,453 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/.tmp/table/db1df5bc1ca840d48e6c041e3b5aadc7 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/table/db1df5bc1ca840d48e6c041e3b5aadc7 2024-11-27T19:15:59,458 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/table/db1df5bc1ca840d48e6c041e3b5aadc7, entries=13, sequenceid=59, filesize=5.7 K 2024-11-27T19:15:59,459 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1304ms, sequenceid=59, compaction requested=false 2024-11-27T19:15:59,463 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2024-11-27T19:15:59,464 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2024-11-27T19:15:59,464 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T19:15:59,464 INFO [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T19:15:59,464 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732734958155Running coprocessor pre-close hooks at 1732734958155Disabling compacts and flushes for region at 1732734958155Disabling writes for close at 1732734958155Obtaining lock to block concurrent updates at 1732734958156 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732734958156Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13486, getHeapSize=23896, getOffHeapSize=0, getCellsCount=125 at 1732734958156Flushing stores of hbase:meta,,1.1588230740 at 1732734958156Flushing 1588230740/info: creating writer at 1732734958156Flushing 1588230740/info: appending metadata at 1732734958178 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732734958178Flushing 1588230740/ns: creating writer at 1732734958587 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732734958603 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732734958603Flushing 1588230740/table: creating writer at 1732734959014 (+411 ms)Flushing 1588230740/table: appending metadata at 1732734959031 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732734959031Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cf7debd: reopening flushed file at 1732734959442 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46f7a6a: reopening flushed file at 1732734959448 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5981e289: reopening flushed file at 1732734959453 (+5 ms)Finished flush of dataSize ~13.17 KB/13486, heapSize ~23.10 KB/23656, currentSize=0 B/0 for 1588230740 in 1304ms, sequenceid=59, compaction requested=false at 1732734959459 (+6 ms)Writing region close event to WAL at 1732734959460 (+1 ms)Running coprocessor post-close hooks at 1732734959463 (+3 ms)Closed at 1732734959464 (+1 ms) 2024-11-27T19:15:59,464 DEBUG [RS_CLOSE_META-regionserver/eb850308c015:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T19:15:59,557 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(976): stopping server eb850308c015,41349,1732734853907; all regions closed. 2024-11-27T19:15:59,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741956_1132 (size=3260) 2024-11-27T19:15:59,566 DEBUG [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1256): Moved 3 WAL file(s) to /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs 2024-11-27T19:15:59,566 INFO [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb850308c015%2C41349%2C1732734853907.meta:.meta(num 1732734946753) 2024-11-27T19:15:59,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741954_1130 (size=1037) 2024-11-27T19:15:59,570 DEBUG [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs 2024-11-27T19:15:59,570 INFO [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-0:(num 1732734946718) 2024-11-27T19:15:59,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741955_1131 (size=5275) 2024-11-27T19:15:59,574 DEBUG [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/oldWALs 2024-11-27T19:15:59,574 INFO [RS:0;eb850308c015:41349 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb850308c015%2C41349%2C1732734853907.eb850308c015%2C41349%2C1732734853907.regiongroup-1:(num 1732734946736) 2024-11-27T19:15:59,574 DEBUG [RS:0;eb850308c015:41349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:59,574 INFO [RS:0;eb850308c015:41349 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T19:15:59,574 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T19:15:59,574 INFO [RS:0;eb850308c015:41349 {}] hbase.ChoreService(370): Chore service for: regionserver/eb850308c015:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T19:15:59,575 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T19:15:59,575 INFO [regionserver/eb850308c015:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T19:15:59,575 INFO [RS:0;eb850308c015:41349 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41349 2024-11-27T19:15:59,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rs/eb850308c015,41349,1732734853907 2024-11-27T19:15:59,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2024-11-27T19:15:59,579 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T19:15:59,583 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eb850308c015,41349,1732734853907] 2024-11-27T19:15:59,584 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /1/draining/eb850308c015,41349,1732734853907 already deleted, retry=false 2024-11-27T19:15:59,584 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eb850308c015,41349,1732734853907 expired; onlineServers=0 2024-11-27T19:15:59,584 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eb850308c015,39035,1732734852191' ***** 2024-11-27T19:15:59,584 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T19:15:59,584 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(468): Stop info server 2024-11-27T19:15:59,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/master 2024-11-27T19:15:59,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2024-11-27T19:15:59,590 INFO [M:0;eb850308c015:39035 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5c685ba8{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/master} 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] server.AbstractConnector(383): Stopped ServerConnector@10f9ba48{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@65507143{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7c1597dd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T19:15:59,591 INFO [M:0;eb850308c015:39035 {}] master.HMaster(1770): Stopping master jetty server 2024-11-27T19:15:59,592 INFO [M:0;eb850308c015:39035 {}] server.AbstractConnector(383): Stopped ServerConnector@1fed14f9{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2024-11-27T19:15:59,593 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T19:15:59,593 DEBUG [M:0;eb850308c015:39035 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T19:15:59,593 INFO [M:0;eb850308c015:39035 {}] master.BackupLogCleaner(194): Stopping BackupLogCleaner 2024-11-27T19:15:59,593 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T19:15:59,593 INFO [M:0;eb850308c015:39035 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;eb850308c015:39035. 2024-11-27T19:15:59,593 DEBUG [M:0;eb850308c015:39035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.backup.BackupHFileCleaner.stop(BackupHFileCleaner.java:156) at org.apache.hadoop.hbase.master.cleaner.CleanerChore.cleanup(CleanerChore.java:444) at org.apache.hadoop.hbase.master.cleaner.HFileCleaner.cleanup(HFileCleaner.java:239) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:333) at org.apache.hadoop.hbase.ScheduledChore.shutdown(ScheduledChore.java:322) at org.apache.hadoop.hbase.master.HMaster.stopChores(HMaster.java:1923) at org.apache.hadoop.hbase.HBaseServerBase.stopChoreService(HBaseServerBase.java:439) at org.apache.hadoop.hbase.master.HMaster.stopServiceThreads(HMaster.java:1777) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:631) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T19:15:59,593 DEBUG [M:0;eb850308c015:39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:59,593 DEBUG [master/eb850308c015:0:becomeActiveMaster-HFileCleaner.small.0-1732734856302 {}] cleaner.HFileCleaner(306): Exit Thread[master/eb850308c015:0:becomeActiveMaster-HFileCleaner.small.0-1732734856302,5,FailOnTimeoutGroup] 2024-11-27T19:15:59,593 DEBUG [M:0;eb850308c015:39035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T19:15:59,593 DEBUG [master/eb850308c015:0:becomeActiveMaster-HFileCleaner.large.0-1732734856302 {}] cleaner.HFileCleaner(306): Exit Thread[master/eb850308c015:0:becomeActiveMaster-HFileCleaner.large.0-1732734856302,5,FailOnTimeoutGroup] 2024-11-27T19:15:59,593 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T19:15:59,593 DEBUG [M:0;eb850308c015:39035 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T19:15:59,593 INFO [M:0;eb850308c015:39035 {}] hbase.ChoreService(370): Chore service for: master/eb850308c015:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T19:15:59,594 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T19:15:59,594 DEBUG [M:0;eb850308c015:39035 {}] master.HMaster(1795): Stopping service threads 2024-11-27T19:15:59,594 INFO [M:0;eb850308c015:39035 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T19:15:59,594 INFO [M:0;eb850308c015:39035 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T19:15:59,594 INFO [M:0;eb850308c015:39035 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T19:15:59,594 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T19:15:59,595 DEBUG [M:0;eb850308c015:39035 {}] zookeeper.ZKUtil(347): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Unable to get data of znode /1/master because node does not exist (not an error) 2024-11-27T19:15:59,595 WARN [M:0;eb850308c015:39035 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T19:15:59,596 INFO [M:0;eb850308c015:39035 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/.lastflushedseqids 2024-11-27T19:15:59,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741981_1157 (size=504) 2024-11-27T19:15:59,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T19:15:59,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41349-0x1015ad983930001, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T19:15:59,683 INFO [RS:0;eb850308c015:41349 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T19:15:59,684 INFO [RS:0;eb850308c015:41349 {}] regionserver.HRegionServer(1031): Exiting; stopping=eb850308c015,41349,1732734853907; zookeeper connection closed. 2024-11-27T19:15:59,684 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2cb15db2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2cb15db2 2024-11-27T19:15:59,685 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T19:15:59,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-jobtracker.properties,hadoop-metrics2.properties 2024-11-27T19:16:00,009 INFO [M:0;eb850308c015:39035 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T19:16:00,009 INFO [M:0;eb850308c015:39035 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T19:16:00,009 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T19:16:00,022 INFO [M:0;eb850308c015:39035 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:16:00,023 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:16:00,023 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T19:16:00,023 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:16:00,023 INFO [M:0;eb850308c015:39035 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=318.88 KB heapSize=374.85 KB 2024-11-27T19:16:00,038 DEBUG [M:0;eb850308c015:39035 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a937bbe111a24c2ea1cfaf1f2bb8a05e is 82, key is hbase:meta,,1/info:regioninfo/1732734857198/Put/seqid=0 2024-11-27T19:16:00,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741982_1158 (size=5672) 2024-11-27T19:16:00,443 INFO [M:0;eb850308c015:39035 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a937bbe111a24c2ea1cfaf1f2bb8a05e 2024-11-27T19:16:00,464 DEBUG [M:0;eb850308c015:39035 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2475b9ba52a476bb4ca87090c9b7515 is 2182, key is \x00\x00\x00\x00\x00\x00\x00*/proc:d/1732734906464/Put/seqid=0 2024-11-27T19:16:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741983_1159 (size=26490) 2024-11-27T19:16:00,869 INFO [M:0;eb850308c015:39035 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=318.33 KB at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2475b9ba52a476bb4ca87090c9b7515 2024-11-27T19:16:00,874 INFO [M:0;eb850308c015:39035 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2475b9ba52a476bb4ca87090c9b7515 2024-11-27T19:16:00,889 DEBUG [M:0;eb850308c015:39035 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e0fe0a6566e4f73903dfac008229d19 is 69, key is eb850308c015,41349,1732734853907/rs:state/1732734856398/Put/seqid=0 2024-11-27T19:16:00,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741984_1160 (size=5156) 2024-11-27T19:16:01,294 INFO [M:0;eb850308c015:39035 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=667 (bloomFilter=true), to=hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e0fe0a6566e4f73903dfac008229d19 2024-11-27T19:16:01,299 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a937bbe111a24c2ea1cfaf1f2bb8a05e as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a937bbe111a24c2ea1cfaf1f2bb8a05e 2024-11-27T19:16:01,303 INFO [M:0;eb850308c015:39035 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a937bbe111a24c2ea1cfaf1f2bb8a05e, entries=8, sequenceid=667, filesize=5.5 K 2024-11-27T19:16:01,304 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b2475b9ba52a476bb4ca87090c9b7515 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b2475b9ba52a476bb4ca87090c9b7515 2024-11-27T19:16:01,308 INFO [M:0;eb850308c015:39035 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2475b9ba52a476bb4ca87090c9b7515 2024-11-27T19:16:01,309 INFO [M:0;eb850308c015:39035 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b2475b9ba52a476bb4ca87090c9b7515, entries=69, sequenceid=667, filesize=25.9 K 2024-11-27T19:16:01,309 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e0fe0a6566e4f73903dfac008229d19 as hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e0fe0a6566e4f73903dfac008229d19 2024-11-27T19:16:01,313 INFO [M:0;eb850308c015:39035 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44785/user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e0fe0a6566e4f73903dfac008229d19, entries=1, sequenceid=667, filesize=5.0 K 2024-11-27T19:16:01,314 INFO [M:0;eb850308c015:39035 {}] regionserver.HRegion(3140): Finished flush of dataSize ~318.88 KB/326535, heapSize ~374.55 KB/383544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=667, compaction requested=false 2024-11-27T19:16:01,316 INFO [M:0;eb850308c015:39035 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T19:16:01,316 DEBUG [M:0;eb850308c015:39035 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732734960009Disabling compacts and flushes for region at 1732734960009Disabling writes for close at 1732734960023 (+14 ms)Obtaining lock to block concurrent updates at 1732734960023Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732734960023Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=326535, getHeapSize=383784, getOffHeapSize=0, getCellsCount=746 at 1732734960023Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732734960024 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732734960024Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732734960038 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732734960038Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732734960448 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732734960464 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732734960464Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732734960874 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732734960889 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732734960889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b39f4cb: reopening flushed file at 1732734961298 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ef04d4: reopening flushed file at 1732734961304 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ef61986: reopening flushed file at 1732734961309 (+5 ms)Finished flush of dataSize ~318.88 KB/326535, heapSize ~374.55 KB/383544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=667, compaction requested=false at 1732734961314 (+5 ms)Writing region close event to WAL at 1732734961316 (+2 ms)Closed at 1732734961316 2024-11-27T19:16:01,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45013 is added to blk_1073741830_1006 (size=370625) 2024-11-27T19:16:01,318 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c67ea052-75c8-da6d-9d50-83f4a54228e9/MasterData/WALs/eb850308c015,39035,1732734852191/eb850308c015%2C39035%2C1732734852191.eb850308c015%2C39035%2C1732734852191.regiongroup-0.1732734855632 not finished, retry = 0 2024-11-27T19:16:01,420 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T19:16:01,420 INFO [M:0;eb850308c015:39035 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T19:16:01,420 INFO [M:0;eb850308c015:39035 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2024-11-27T19:16:01,420 INFO [M:0;eb850308c015:39035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39035 2024-11-27T19:16:01,420 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T19:16:01,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T19:16:01,522 INFO [M:0;eb850308c015:39035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T19:16:01,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39035-0x1015ad983930000, quorum=127.0.0.1:63134, baseZNode=/1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T19:16:01,533 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@cb44ad4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T19:16:01,535 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d3ca681{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,536 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8cd91a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,536 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@526a6450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,538 WARN [BP-496223468-172.17.0.2-1732734849103 heartbeating to localhost/127.0.0.1:44785 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T19:16:01,538 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T19:16:01,538 WARN [BP-496223468-172.17.0.2-1732734849103 heartbeating to localhost/127.0.0.1:44785 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-496223468-172.17.0.2-1732734849103 (Datanode Uuid 927051a4-f8bb-4e6a-8bb8-ec34dd96f242) service to localhost/127.0.0.1:44785 2024-11-27T19:16:01,538 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T19:16:01,540 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data1/current/BP-496223468-172.17.0.2-1732734849103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T19:16:01,540 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/cluster_969a69f0-150f-f172-e692-32308e83586a/data/data2/current/BP-496223468-172.17.0.2-1732734849103 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T19:16:01,540 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T19:16:01,547 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@38e78c7b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T19:16:01,547 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c336369{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,547 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,547 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b22dca{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,548 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f045f3e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,556 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T19:16:01,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-27T19:16:01,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2377): Stopping mini mapreduce cluster... 2024-11-27T19:16:01,612 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@223872ea{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-27T19:16:01,613 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53a6a2d1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,613 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,613 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ba88f54{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,613 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@db00356{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,622 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10a4b359{node,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/node} 2024-11-27T19:16:01,622 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@37d45da6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,622 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,622 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b50b82e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,622 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@496af81f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,636 ERROR [Thread[Thread-270,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-27T19:16:01,636 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a3f7689{cluster,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/cluster} 2024-11-27T19:16:01,637 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@567287a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,637 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,637 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31795375{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,637 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ccdffea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,640 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2024-11-27T19:16:01,644 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(72): Returning, interrupted : java.lang.InterruptedException 2024-11-27T19:16:01,645 ERROR [ResourceManager Event Processor Monitor {}] resourcemanager.ResourceManager$SchedulerEventDispatcher$EventProcessorMonitor(1193): Returning, interrupted : java.lang.InterruptedException: sleep interrupted 2024-11-27T19:16:01,646 ERROR [Thread[Thread-293,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-27T19:16:01,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@73c48ef9{jobhistory,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/jobhistory} 2024-11-27T19:16:01,649 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15a81a3d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T19:16:01,649 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T19:16:01,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a60e5b2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-yarn-common/3.4.1/hadoop-yarn-common-3.4.1.jar!/webapps/static,STOPPED} 2024-11-27T19:16:01,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@171eacd3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-backup/target/test-data/5b85186f-0dcb-fef7-d0b2-26d0a6076009/hadoop.log.dir/,STOPPED} 2024-11-27T19:16:01,652 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(852): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2024-11-27T19:16:01,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2380): Mini mapreduce cluster stopped