2024-12-04 15:14:19,468 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@35841320 2024-12-04 15:14:19,486 main DEBUG Took 0.014861 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 15:14:19,487 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 15:14:19,488 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 15:14:19,489 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 15:14:19,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,507 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 15:14:19,549 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,551 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,553 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,557 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,557 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,559 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,562 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,565 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,567 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,569 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,572 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,575 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,578 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,582 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 15:14:19,585 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,585 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 15:14:19,588 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 15:14:19,591 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 15:14:19,593 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 15:14:19,594 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 15:14:19,596 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 15:14:19,597 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 15:14:19,610 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 15:14:19,613 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 15:14:19,615 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 15:14:19,616 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 15:14:19,617 main DEBUG createAppenders(={Console}) 2024-12-04 15:14:19,618 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@35841320 initialized 2024-12-04 15:14:19,619 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@35841320 2024-12-04 15:14:19,620 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@35841320 OK. 2024-12-04 15:14:19,621 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 15:14:19,621 main DEBUG OutputStream closed 2024-12-04 15:14:19,621 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 15:14:19,622 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 15:14:19,622 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@bd4dc25 OK 2024-12-04 15:14:19,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 15:14:19,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 15:14:19,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 15:14:19,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 15:14:19,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 15:14:19,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 15:14:19,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 15:14:19,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 15:14:19,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 15:14:19,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 15:14:19,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 15:14:19,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 15:14:19,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 15:14:19,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 15:14:19,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 15:14:19,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 15:14:19,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 15:14:19,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 15:14:19,730 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 15:14:19,730 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6ce86ce1) with optional ClassLoader: null 2024-12-04 15:14:19,730 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 15:14:19,731 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6ce86ce1] started OK. 2024-12-04T15:14:19,950 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575 2024-12-04 15:14:19,953 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 15:14:19,954 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T15:14:19,979 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-04T15:14:20,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T15:14:20,015 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a, deleteOnExit=true 2024-12-04T15:14:20,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T15:14:20,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/test.cache.data in system properties and HBase conf 2024-12-04T15:14:20,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T15:14:20,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.log.dir in system properties and HBase conf 2024-12-04T15:14:20,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T15:14:20,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T15:14:20,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T15:14:20,108 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T15:14:20,204 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T15:14:20,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:14:20,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T15:14:20,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T15:14:20,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:14:20,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T15:14:20,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T15:14:20,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T15:14:20,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:14:20,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T15:14:20,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/nfs.dump.dir in system properties and HBase conf 2024-12-04T15:14:20,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/java.io.tmpdir in system properties and HBase conf 2024-12-04T15:14:20,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T15:14:20,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T15:14:20,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T15:14:21,252 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T15:14:21,346 INFO [Time-limited test {}] log.Log(170): Logging initialized @2764ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T15:14:21,430 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:14:21,488 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:14:21,507 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:14:21,507 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:14:21,508 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:14:21,519 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:14:21,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:14:21,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T15:14:21,707 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1441d181{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/java.io.tmpdir/jetty-localhost-42365-hadoop-hdfs-3_4_1-tests_jar-_-any-11766599509983351507/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T15:14:21,717 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:42365} 2024-12-04T15:14:21,717 INFO [Time-limited test {}] server.Server(415): Started @3136ms 2024-12-04T15:14:22,223 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T15:14:22,233 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T15:14:22,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T15:14:22,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T15:14:22,235 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T15:14:22,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@698dfd8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.log.dir/,AVAILABLE} 2024-12-04T15:14:22,237 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62b38b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T15:14:22,360 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e80f2cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/java.io.tmpdir/jetty-localhost-34415-hadoop-hdfs-3_4_1-tests_jar-_-any-6490273207642591650/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T15:14:22,360 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:34415} 2024-12-04T15:14:22,361 INFO [Time-limited test {}] server.Server(415): Started @3779ms 2024-12-04T15:14:22,414 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T15:14:23,304 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/data/data2/current/BP-1046799081-172.17.0.3-1733325260798/current, will proceed with Du for space computation calculation, 2024-12-04T15:14:23,304 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/data/data1/current/BP-1046799081-172.17.0.3-1733325260798/current, will proceed with Du for space computation calculation, 2024-12-04T15:14:23,342 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T15:14:23,409 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x58c63dfa9927115b with lease ID 0xd943913f619fd530: Processing first storage report for DS-8d8134b6-6764-4520-b7d4-17362b1b8067 from datanode DatanodeRegistration(127.0.0.1:40245, datanodeUuid=8f9716f7-691d-4710-be47-7687cc4aaa4b, infoPort=33113, infoSecurePort=0, ipcPort=40827, storageInfo=lv=-57;cid=testClusterID;nsid=1727295528;c=1733325260798) 2024-12-04T15:14:23,410 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x58c63dfa9927115b with lease ID 0xd943913f619fd530: from storage DS-8d8134b6-6764-4520-b7d4-17362b1b8067 node DatanodeRegistration(127.0.0.1:40245, datanodeUuid=8f9716f7-691d-4710-be47-7687cc4aaa4b, infoPort=33113, infoSecurePort=0, ipcPort=40827, storageInfo=lv=-57;cid=testClusterID;nsid=1727295528;c=1733325260798), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T15:14:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x58c63dfa9927115b with lease ID 0xd943913f619fd530: Processing first storage report for DS-16aa4b58-f73b-42cb-8be0-451a533b29ee from datanode DatanodeRegistration(127.0.0.1:40245, datanodeUuid=8f9716f7-691d-4710-be47-7687cc4aaa4b, infoPort=33113, infoSecurePort=0, ipcPort=40827, storageInfo=lv=-57;cid=testClusterID;nsid=1727295528;c=1733325260798) 2024-12-04T15:14:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x58c63dfa9927115b with lease ID 0xd943913f619fd530: from storage DS-16aa4b58-f73b-42cb-8be0-451a533b29ee node DatanodeRegistration(127.0.0.1:40245, datanodeUuid=8f9716f7-691d-4710-be47-7687cc4aaa4b, infoPort=33113, infoSecurePort=0, ipcPort=40827, storageInfo=lv=-57;cid=testClusterID;nsid=1727295528;c=1733325260798), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T15:14:23,453 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575 2024-12-04T15:14:23,536 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/zookeeper_0, clientPort=64157, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T15:14:23,549 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64157 2024-12-04T15:14:23,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:23,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:23,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741825_1001 (size=7) 2024-12-04T15:14:24,223 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb with version=8 2024-12-04T15:14:24,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/hbase-staging 2024-12-04T15:14:24,360 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T15:14:24,701 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c6f2e1d35cf1:0 server-side Connection retries=6 2024-12-04T15:14:24,718 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:24,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:24,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T15:14:24,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:24,741 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T15:14:25,008 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T15:14:25,091 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T15:14:25,104 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T15:14:25,109 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T15:14:25,152 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8182 (auto-detected) 2024-12-04T15:14:25,154 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-04T15:14:25,190 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46533 2024-12-04T15:14:25,227 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46533 connecting to ZooKeeper ensemble=127.0.0.1:64157 2024-12-04T15:14:25,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:465330x0, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T15:14:25,348 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46533-0x101a21658430000 connected 2024-12-04T15:14:25,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:25,521 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:25,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:14:25,546 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb, hbase.cluster.distributed=false 2024-12-04T15:14:25,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T15:14:25,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46533 2024-12-04T15:14:25,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46533 2024-12-04T15:14:25,611 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46533 2024-12-04T15:14:25,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46533 2024-12-04T15:14:25,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46533 2024-12-04T15:14:25,866 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c6f2e1d35cf1:0 server-side Connection retries=6 2024-12-04T15:14:25,869 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:25,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:25,870 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T15:14:25,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T15:14:25,880 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T15:14:25,889 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T15:14:25,902 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T15:14:25,904 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37535 2024-12-04T15:14:25,906 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37535 connecting to ZooKeeper ensemble=127.0.0.1:64157 2024-12-04T15:14:25,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:25,942 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375350x0, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T15:14:26,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:375350x0, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:14:26,018 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37535-0x101a21658430001 connected 2024-12-04T15:14:26,022 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T15:14:26,053 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T15:14:26,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T15:14:26,099 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T15:14:26,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37535 2024-12-04T15:14:26,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37535 2024-12-04T15:14:26,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37535 2024-12-04T15:14:26,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37535 2024-12-04T15:14:26,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37535 2024-12-04T15:14:26,241 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c6f2e1d35cf1:46533 2024-12-04T15:14:26,247 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:26,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T15:14:26,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T15:14:26,295 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:26,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T15:14:26,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:26,376 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T15:14:26,386 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c6f2e1d35cf1,46533,1733325264441 from backup master directory 2024-12-04T15:14:26,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:26,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T15:14:26,399 WARN [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T15:14:26,400 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:26,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T15:14:26,405 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T15:14:26,408 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T15:14:26,589 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/hbase.id] with ID: 6aa3465c-d757-4950-ad0d-9868eb0e70b4 2024-12-04T15:14:26,589 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/.tmp/hbase.id 2024-12-04T15:14:26,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741826_1002 (size=42) 2024-12-04T15:14:26,679 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/.tmp/hbase.id]:[hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/hbase.id] 2024-12-04T15:14:26,984 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T15:14:26,999 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T15:14:27,080 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 70ms. 2024-12-04T15:14:27,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:27,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:27,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741827_1003 (size=196) 2024-12-04T15:14:27,286 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:14:27,292 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T15:14:27,328 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:27,335 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:14:27,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741828_1004 (size=1189) 2024-12-04T15:14:27,866 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store 2024-12-04T15:14:27,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741829_1005 (size=34) 2024-12-04T15:14:28,020 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T15:14:28,029 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:28,030 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T15:14:28,031 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:14:28,031 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:14:28,038 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T15:14:28,038 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:14:28,038 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:14:28,043 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733325268030Disabling compacts and flushes for region at 1733325268030Disabling writes for close at 1733325268038 (+8 ms)Writing region close event to WAL at 1733325268038Closed at 1733325268038 2024-12-04T15:14:28,055 WARN [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/.initializing 2024-12-04T15:14:28,055 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/WALs/c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:28,083 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T15:14:28,137 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f2e1d35cf1%2C46533%2C1733325264441, suffix=, logDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/WALs/c6f2e1d35cf1,46533,1733325264441, archiveDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/oldWALs, maxLogs=10 2024-12-04T15:14:28,204 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/WALs/c6f2e1d35cf1,46533,1733325264441/c6f2e1d35cf1%2C46533%2C1733325264441.1733325268166, exclude list is [], retry=0 2024-12-04T15:14:28,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40245,DS-8d8134b6-6764-4520-b7d4-17362b1b8067,DISK] 2024-12-04T15:14:28,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T15:14:28,349 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/WALs/c6f2e1d35cf1,46533,1733325264441/c6f2e1d35cf1%2C46533%2C1733325264441.1733325268166 2024-12-04T15:14:28,368 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113)] 2024-12-04T15:14:28,368 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:14:28,369 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:28,400 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,405 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,613 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T15:14:28,625 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:28,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:28,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T15:14:28,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:28,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:28,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T15:14:28,675 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:28,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:28,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T15:14:28,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:28,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:28,701 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,712 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,715 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,736 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,738 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,749 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:14:28,771 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T15:14:28,811 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:14:28,818 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65314732, jitterRate=-0.026734650135040283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:14:28,834 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733325268468Initializing all the Stores at 1733325268487 (+19 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325268488 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325268496 (+8 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325268498 (+2 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325268502 (+4 ms)Cleaning up temporary data from old regions at 1733325268738 (+236 ms)Region opened successfully at 1733325268833 (+95 ms) 2024-12-04T15:14:28,843 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T15:14:28,950 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3218b9d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6f2e1d35cf1/172.17.0.3:0 2024-12-04T15:14:29,046 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T15:14:29,072 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T15:14:29,072 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T15:14:29,086 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T15:14:29,095 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-12-04T15:14:29,105 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-04T15:14:29,105 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T15:14:29,223 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T15:14:29,255 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T15:14:29,336 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T15:14:29,346 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T15:14:29,356 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T15:14:29,368 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T15:14:29,379 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T15:14:29,392 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T15:14:29,403 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T15:14:29,410 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T15:14:29,424 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T15:14:29,490 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T15:14:29,500 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T15:14:29,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T15:14:29,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:29,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T15:14:29,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:29,542 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c6f2e1d35cf1,46533,1733325264441, sessionid=0x101a21658430000, setting cluster-up flag (Was=false) 2024-12-04T15:14:29,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:29,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:29,648 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T15:14:29,672 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:29,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:29,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:30,029 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T15:14:30,052 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:30,069 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T15:14:30,173 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(746): ClusterId : 6aa3465c-d757-4950-ad0d-9868eb0e70b4 2024-12-04T15:14:30,177 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T15:14:30,224 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T15:14:30,224 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T15:14:30,247 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T15:14:30,248 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35cc5b2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c6f2e1d35cf1/172.17.0.3:0 2024-12-04T15:14:30,293 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c6f2e1d35cf1:37535 2024-12-04T15:14:30,299 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T15:14:30,310 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T15:14:30,311 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T15:14:30,311 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T15:14:30,325 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6f2e1d35cf1,46533,1733325264441 with port=37535, startcode=1733325265751 2024-12-04T15:14:30,348 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T15:14:30,351 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T15:14:30,364 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T15:14:30,385 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c6f2e1d35cf1,46533,1733325264441 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T15:14:30,441 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c6f2e1d35cf1:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:14:30,442 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c6f2e1d35cf1:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:14:30,442 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c6f2e1d35cf1:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:14:30,442 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c6f2e1d35cf1:0, corePoolSize=5, maxPoolSize=5 2024-12-04T15:14:30,443 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c6f2e1d35cf1:0, corePoolSize=10, maxPoolSize=10 2024-12-04T15:14:30,443 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,443 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c6f2e1d35cf1:0, corePoolSize=2, maxPoolSize=2 2024-12-04T15:14:30,443 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,476 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39581, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T15:14:30,492 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:14:30,493 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T15:14:30,495 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46533 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:30,509 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733325300509 2024-12-04T15:14:30,512 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T15:14:30,514 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T15:14:30,516 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:30,516 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T15:14:30,518 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T15:14:30,519 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T15:14:30,519 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T15:14:30,520 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T15:14:30,524 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,535 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T15:14:30,537 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T15:14:30,537 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T15:14:30,574 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T15:14:30,575 WARN [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T15:14:30,576 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T15:14:30,577 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T15:14:30,595 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.large.0-1733325270589,5,FailOnTimeoutGroup] 2024-12-04T15:14:30,601 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.small.0-1733325270595,5,FailOnTimeoutGroup] 2024-12-04T15:14:30,601 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,602 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T15:14:30,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741831_1007 (size=1321) 2024-12-04T15:14:30,607 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,609 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,612 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T15:14:30,613 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb 2024-12-04T15:14:30,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741832_1008 (size=32) 2024-12-04T15:14:30,683 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(2659): reportForDuty to master=c6f2e1d35cf1,46533,1733325264441 with port=37535, startcode=1733325265751 2024-12-04T15:14:30,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:30,695 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46533 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:30,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46533 {}] master.ServerManager(517): Registering regionserver=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:30,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T15:14:30,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T15:14:30,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:30,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:30,719 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T15:14:30,727 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb 2024-12-04T15:14:30,727 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41653 2024-12-04T15:14:30,728 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T15:14:30,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T15:14:30,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:30,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T15:14:30,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:30,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T15:14:30,763 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] zookeeper.ZKUtil(111): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:30,763 WARN [RS:0;c6f2e1d35cf1:37535 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T15:14:30,763 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:14:30,764 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:30,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T15:14:30,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:30,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:30,783 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T15:14:30,785 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c6f2e1d35cf1,37535,1733325265751] 2024-12-04T15:14:30,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T15:14:30,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:30,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:30,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T15:14:30,812 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740 2024-12-04T15:14:30,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740 2024-12-04T15:14:30,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T15:14:30,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T15:14:30,862 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T15:14:30,864 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:14:30,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T15:14:30,899 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:14:30,903 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T15:14:30,905 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71397283, jitterRate=0.06390242278575897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:14:30,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733325270690Initializing all the Stores at 1733325270702 (+12 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325270702Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325270703 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325270703Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325270703Cleaning up temporary data from old regions at 1733325270853 (+150 ms)Region opened successfully at 1733325270911 (+58 ms) 2024-12-04T15:14:30,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T15:14:30,912 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T15:14:30,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T15:14:30,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T15:14:30,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T15:14:30,915 INFO [RS:0;c6f2e1d35cf1:37535 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T15:14:30,915 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,917 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T15:14:30,920 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T15:14:30,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733325270911Disabling compacts and flushes for region at 1733325270911Disabling writes for close at 1733325270912 (+1 ms)Writing region close event to WAL at 1733325270919 (+7 ms)Closed at 1733325270920 (+1 ms) 2024-12-04T15:14:30,929 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:14:30,930 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T15:14:30,937 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T15:14:30,939 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,940 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,940 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,940 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T15:14:30,943 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,943 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,943 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c6f2e1d35cf1:0, corePoolSize=2, maxPoolSize=2 2024-12-04T15:14:30,943 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c6f2e1d35cf1:0, corePoolSize=1, maxPoolSize=1 2024-12-04T15:14:30,944 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c6f2e1d35cf1:0, corePoolSize=3, maxPoolSize=3 2024-12-04T15:14:30,945 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c6f2e1d35cf1:0, corePoolSize=3, maxPoolSize=3 2024-12-04T15:14:30,956 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T15:14:30,965 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T15:14:30,983 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,983 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,999 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,999 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:30,999 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:31,000 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,37535,1733325265751-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T15:14:31,042 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T15:14:31,045 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,37535,1733325265751-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:31,046 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:31,046 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.Replication(171): c6f2e1d35cf1,37535,1733325265751 started 2024-12-04T15:14:31,082 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:31,083 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1482): Serving as c6f2e1d35cf1,37535,1733325265751, RpcServer on c6f2e1d35cf1/172.17.0.3:37535, sessionid=0x101a21658430001 2024-12-04T15:14:31,085 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T15:14:31,085 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:31,086 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6f2e1d35cf1,37535,1733325265751' 2024-12-04T15:14:31,086 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T15:14:31,096 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T15:14:31,107 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T15:14:31,109 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T15:14:31,109 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:31,109 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c6f2e1d35cf1,37535,1733325265751' 2024-12-04T15:14:31,109 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T15:14:31,113 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T15:14:31,115 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T15:14:31,115 INFO [RS:0;c6f2e1d35cf1:37535 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T15:14:31,115 INFO [RS:0;c6f2e1d35cf1:37535 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T15:14:31,117 WARN [c6f2e1d35cf1:46533 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T15:14:31,225 INFO [RS:0;c6f2e1d35cf1:37535 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T15:14:31,243 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f2e1d35cf1%2C37535%2C1733325265751, suffix=, logDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751, archiveDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs, maxLogs=32 2024-12-04T15:14:31,267 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.1733325271247, exclude list is [], retry=0 2024-12-04T15:14:31,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40245,DS-8d8134b6-6764-4520-b7d4-17362b1b8067,DISK] 2024-12-04T15:14:31,291 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.1733325271247 2024-12-04T15:14:31,292 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113)] 2024-12-04T15:14:31,372 DEBUG [c6f2e1d35cf1:46533 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T15:14:31,403 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:31,417 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6f2e1d35cf1,37535,1733325265751, state=OPENING 2024-12-04T15:14:31,429 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T15:14:31,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:31,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:31,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T15:14:31,461 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T15:14:31,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T15:14:31,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c6f2e1d35cf1,37535,1733325265751}] 2024-12-04T15:14:31,712 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T15:14:31,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51469, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T15:14:31,775 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T15:14:31,787 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:14:31,788 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T15:14:31,797 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f2e1d35cf1%2C37535%2C1733325265751.meta, suffix=.meta, logDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751, archiveDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs, maxLogs=32 2024-12-04T15:14:31,838 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.meta.1733325271804.meta, exclude list is [], retry=0 2024-12-04T15:14:31,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40245,DS-8d8134b6-6764-4520-b7d4-17362b1b8067,DISK] 2024-12-04T15:14:31,877 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.meta.1733325271804.meta 2024-12-04T15:14:31,878 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113)] 2024-12-04T15:14:31,879 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:14:31,882 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T15:14:31,896 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T15:14:31,909 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T15:14:31,927 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T15:14:31,931 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:31,932 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T15:14:31,932 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T15:14:31,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T15:14:31,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T15:14:31,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:31,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:31,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T15:14:31,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T15:14:31,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:31,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:31,971 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T15:14:31,983 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T15:14:31,983 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:31,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:31,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T15:14:32,004 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T15:14:32,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:32,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T15:14:32,015 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T15:14:32,023 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740 2024-12-04T15:14:32,034 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740 2024-12-04T15:14:32,043 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T15:14:32,043 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T15:14:32,045 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T15:14:32,057 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T15:14:32,061 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63895811, jitterRate=-0.04787822067737579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T15:14:32,061 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T15:14:32,063 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733325271933Writing region info on filesystem at 1733325271935 (+2 ms)Initializing all the Stores at 1733325271943 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325271944 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325271945 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325271945Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733325271945Cleaning up temporary data from old regions at 1733325272044 (+99 ms)Running coprocessor post-open hooks at 1733325272061 (+17 ms)Region opened successfully at 1733325272063 (+2 ms) 2024-12-04T15:14:32,083 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733325271678 2024-12-04T15:14:32,118 DEBUG [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T15:14:32,120 INFO [RS_OPEN_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T15:14:32,123 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:32,128 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c6f2e1d35cf1,37535,1733325265751, state=OPEN 2024-12-04T15:14:32,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T15:14:32,202 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T15:14:32,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T15:14:32,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T15:14:32,203 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:32,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T15:14:32,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c6f2e1d35cf1,37535,1733325265751 in 739 msec 2024-12-04T15:14:32,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T15:14:32,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2790 sec 2024-12-04T15:14:32,240 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T15:14:32,240 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T15:14:32,271 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:14:32,273 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=-1] 2024-12-04T15:14:32,311 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:14:32,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58877, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:14:32,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.2160 sec 2024-12-04T15:14:32,385 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733325272385, completionTime=-1 2024-12-04T15:14:32,389 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T15:14:32,389 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T15:14:32,487 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T15:14:32,487 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733325332487 2024-12-04T15:14:32,488 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733325392488 2024-12-04T15:14:32,488 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 98 msec 2024-12-04T15:14:32,493 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,498 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,499 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,501 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c6f2e1d35cf1:46533, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,510 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,511 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,514 DEBUG [master/c6f2e1d35cf1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T15:14:32,565 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 6.164sec 2024-12-04T15:14:32,567 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T15:14:32,569 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T15:14:32,570 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T15:14:32,577 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T15:14:32,578 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T15:14:32,579 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T15:14:32,580 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T15:14:32,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f5b8af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,617 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T15:14:32,617 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T15:14:32,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f2e1d35cf1,46533,-1 for getting cluster id 2024-12-04T15:14:32,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:14:32,638 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T15:14:32,639 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T15:14:32,640 INFO [master/c6f2e1d35cf1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c6f2e1d35cf1,46533,1733325264441-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T15:14:32,692 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6aa3465c-d757-4950-ad0d-9868eb0e70b4' 2024-12-04T15:14:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:14:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6aa3465c-d757-4950-ad0d-9868eb0e70b4" 2024-12-04T15:14:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd2c684, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f2e1d35cf1,46533,-1] 2024-12-04T15:14:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:14:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:14:32,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:14:32,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1cbff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:14:32,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=-1] 2024-12-04T15:14:32,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:14:32,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T15:14:32,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:32,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2297e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c6f2e1d35cf1,46533,-1 for getting cluster id 2024-12-04T15:14:32,828 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T15:14:32,851 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6aa3465c-d757-4950-ad0d-9868eb0e70b4' 2024-12-04T15:14:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T15:14:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6aa3465c-d757-4950-ad0d-9868eb0e70b4" 2024-12-04T15:14:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c6f83a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c6f2e1d35cf1,46533,-1] 2024-12-04T15:14:32,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T15:14:32,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:14:32,886 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T15:14:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127d8b20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T15:14:32,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=932, ProcessCount=11, AvailableMemoryMB=3220 2024-12-04T15:14:33,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T15:14:33,037 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:14:33,049 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76a8cd74 2024-12-04T15:14:33,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T15:14:33,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T15:14:33,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:33,114 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:33,124 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5d1652e7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41c9caa2 2024-12-04T15:14:33,127 WARN [PEWorker-3 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-04T15:14:33,167 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5d1652e7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5d1652e7 to 127.0.0.1:64157 2024-12-04T15:14:33,167 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:33,170 INFO [PEWorker-3 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T15:14:33,204 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication 2024-12-04T15:14:33,214 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T15:14:33,215 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:33,219 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T15:14:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:33,260 DEBUG [PEWorker-3 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication 2024-12-04T15:14:33,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741835_1011 (size=1138) 2024-12-04T15:14:33,347 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 169fab88624a99a661f8d8b6fa142c42, NAME => 'hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb 2024-12-04T15:14:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741836_1012 (size=44) 2024-12-04T15:14:33,415 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:33,416 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing 169fab88624a99a661f8d8b6fa142c42, disabling compactions & flushes 2024-12-04T15:14:33,416 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:33,416 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:33,416 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. after waiting 0 ms 2024-12-04T15:14:33,416 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:33,416 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:33,416 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for 169fab88624a99a661f8d8b6fa142c42: Waiting for close lock at 1733325273416Disabling compacts and flushes for region at 1733325273416Disabling writes for close at 1733325273416Writing region close event to WAL at 1733325273416Closed at 1733325273416 2024-12-04T15:14:33,423 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T15:14:33,430 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1733325273423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733325273423"}]},"ts":"1733325273423"} 2024-12-04T15:14:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:33,455 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T15:14:33,458 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T15:14:33,462 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325273458"}]},"ts":"1733325273458"} 2024-12-04T15:14:33,477 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta 2024-12-04T15:14:33,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=169fab88624a99a661f8d8b6fa142c42, ASSIGN}] 2024-12-04T15:14:33,489 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=169fab88624a99a661f8d8b6fa142c42, ASSIGN 2024-12-04T15:14:33,492 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=169fab88624a99a661f8d8b6fa142c42, ASSIGN; state=OFFLINE, location=c6f2e1d35cf1,37535,1733325265751; forceNewPlan=false, retain=false 2024-12-04T15:14:33,646 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=169fab88624a99a661f8d8b6fa142c42, regionState=OPENING, regionLocation=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:33,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=169fab88624a99a661f8d8b6fa142c42, ASSIGN because future has completed 2024-12-04T15:14:33,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 169fab88624a99a661f8d8b6fa142c42, server=c6f2e1d35cf1,37535,1733325265751}] 2024-12-04T15:14:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:33,876 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:33,876 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T15:14:33,877 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep 2024-12-04T15:14:33,891 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c6f2e1d35cf1%2C37535%2C1733325265751.rep, suffix=, logDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751, archiveDir=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs, maxLogs=32 2024-12-04T15:14:33,930 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.rep.1733325273894, exclude list is [], retry=0 2024-12-04T15:14:33,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40245,DS-8d8134b6-6764-4520-b7d4-17362b1b8067,DISK] 2024-12-04T15:14:33,980 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.rep.1733325273894 2024-12-04T15:14:34,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33113:33113)] 2024-12-04T15:14:34,007 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => 169fab88624a99a661f8d8b6fa142c42, NAME => 'hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42.', STARTKEY => '', ENDKEY => ''} 2024-12-04T15:14:34,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T15:14:34,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. service=MultiRowMutationService 2024-12-04T15:14:34,008 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully. 2024-12-04T15:14:34,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T15:14:34,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,042 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,050 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 169fab88624a99a661f8d8b6fa142c42 columnFamilyName hfileref 2024-12-04T15:14:34,050 DEBUG [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:34,055 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(327): Store=169fab88624a99a661f8d8b6fa142c42/hfileref, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:34,056 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,060 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 169fab88624a99a661f8d8b6fa142c42 columnFamilyName queue 2024-12-04T15:14:34,061 DEBUG [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:34,063 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(327): Store=169fab88624a99a661f8d8b6fa142c42/queue, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:34,064 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,071 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 169fab88624a99a661f8d8b6fa142c42 columnFamilyName sid 2024-12-04T15:14:34,071 DEBUG [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T15:14:34,073 INFO [StoreOpener-169fab88624a99a661f8d8b6fa142c42-1 {}] regionserver.HStore(327): Store=169fab88624a99a661f8d8b6fa142c42/sid, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T15:14:34,074 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,083 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,091 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,119 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T15:14:34,124 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T15:14:34,142 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened 169fab88624a99a661f8d8b6fa142c42; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70494538, jitterRate=0.05045047402381897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T15:14:34,142 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:14:34,143 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for 169fab88624a99a661f8d8b6fa142c42: Running coprocessor pre-open hook at 1733325274009Writing region info on filesystem at 1733325274009Initializing all the Stores at 1733325274027 (+18 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325274027Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325274042 (+15 ms)Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733325274042Cleaning up temporary data from old regions at 1733325274111 (+69 ms)Running coprocessor post-open hooks at 1733325274142 (+31 ms)Region opened successfully at 1733325274143 (+1 ms) 2024-12-04T15:14:34,154 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., pid=7, masterSystemTime=1733325273847 2024-12-04T15:14:34,167 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=169fab88624a99a661f8d8b6fa142c42, regionState=OPEN, openSeqNum=2, regionLocation=c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:34,178 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:34,179 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:14:34,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 169fab88624a99a661f8d8b6fa142c42, server=c6f2e1d35cf1,37535,1733325265751 because future has completed 2024-12-04T15:14:34,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6 2024-12-04T15:14:34,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 169fab88624a99a661f8d8b6fa142c42, server=c6f2e1d35cf1,37535,1733325265751 in 521 msec 2024-12-04T15:14:34,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T15:14:34,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=169fab88624a99a661f8d8b6fa142c42, ASSIGN in 728 msec 2024-12-04T15:14:34,235 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T15:14:34,235 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733325274235"}]},"ts":"1733325274235"} 2024-12-04T15:14:34,251 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta 2024-12-04T15:14:34,254 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T15:14:34,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 1.0830 sec 2024-12-04T15:14:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:34,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=2] 2024-12-04T15:14:34,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-04T15:14:34,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:34,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:14:34,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:34,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8 2024-12-04T15:14:34,687 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:34,819 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:34,824 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T15:14:34,824 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=-1] 2024-12-04T15:14:34,830 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T15:14:34,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42427, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-04T15:14:34,841 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-c6f2e1d35cf1,37535,1733325265751', locateType=CURRENT is [region=hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=2] 2024-12-04T15:14:34,875 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-04T15:14:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-04T15:14:34,899 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:34,914 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:34,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 1.8250 sec 2024-12-04T15:14:34,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-12-04T15:14:34,922 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 396 msec 2024-12-04T15:14:34,936 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:34,940 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3cee4cc5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65752b33 2024-12-04T15:14:34,994 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:34,994 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:35,091 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T15:14:35,092 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T15:14:35,110 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication 2024-12-04T15:14:35,110 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer 2024-12-04T15:14:35,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:14:35,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T15:14:35,114 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T15:14:35,114 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T15:14:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T15:14:35,291 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:35,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:35,311 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:35,313 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0986e3f7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d50f71a 2024-12-04T15:14:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T15:14:35,375 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0986e3f7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0986e3f7 to 127.0.0.1:64157 2024-12-04T15:14:35,375 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T15:14:35,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:35,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-04T15:14:35,601 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:35,605 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:35,605 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:35,606 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:35,606 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:35,608 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x370af7b2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56430fca 2024-12-04T15:14:35,615 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:35,653 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:35,654 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T15:14:35,658 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-04T15:14:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-04T15:14:35,675 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:35,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-04T15:14:35,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 239 msec 2024-12-04T15:14:35,683 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:35,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 390 msec 2024-12-04T15:14:35,693 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:35,696 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6696e4e2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c9edf13 2024-12-04T15:14:35,778 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:35,779 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-04T15:14:35,974 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:35,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:35,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:35,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:36,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:14:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:36,008 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:36,010 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0536acd8 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7dfdc395 2024-12-04T15:14:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:14:36,070 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0536acd8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0536acd8 to 127.0.0.1:64157 2024-12-04T15:14:36,070 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:14:36,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:36,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-04T15:14:36,326 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:36,329 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:36,329 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:36,329 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:36,330 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:36,332 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56b58936 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48a60731 2024-12-04T15:14:36,334 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:14:36,361 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:36,362 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:36,387 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-04T15:14:36,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-04T15:14:36,398 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:36,416 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:36,421 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:14:36,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 419 msec 2024-12-04T15:14:36,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T15:14:36,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 234 msec 2024-12-04T15:14:36,431 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a53f3f1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c007ad 2024-12-04T15:14:36,496 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:36,498 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-04T15:14:36,653 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:36,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:36,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:36,674 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:36,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:36,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-04T15:14:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T15:14:36,691 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:36,694 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x47dce3b1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2786c6d4 2024-12-04T15:14:36,746 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x47dce3b1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x47dce3b1 to 127.0.0.1:64157 2024-12-04T15:14:36,751 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:36,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T15:14:36,841 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication' 2024-12-04T15:14:36,846 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T15:14:36,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-04T15:14:36,934 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:36,947 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:36,947 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:36,947 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:36,948 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:36,953 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b152d45 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67601881 2024-12-04T15:14:36,964 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T15:14:37,039 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:37,040 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:37,050 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-04T15:14:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-04T15:14:37,079 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:37,091 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:37,100 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x648d6c1e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21916a32 2024-12-04T15:14:37,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T15:14:37,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 311 msec 2024-12-04T15:14:37,116 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-04T15:14:37,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 435 msec 2024-12-04T15:14:37,139 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:37,140 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T15:14:37,317 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:37,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:37,321 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:37,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:37,330 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T15:14:37,338 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:37,340 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d1d3518 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5753bf4 2024-12-04T15:14:37,382 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0d1d3518 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d1d3518 to 127.0.0.1:64157 2024-12-04T15:14:37,385 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T15:14:37,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:37,540 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:14:37,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-04T15:14:37,619 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:37,625 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:37,626 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:37,627 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:37,628 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:37,630 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x388552d5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b5bbd88 2024-12-04T15:14:37,632 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T15:14:37,712 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:37,712 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:37,713 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-04T15:14:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-04T15:14:37,728 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:37,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-04T15:14:37,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-04T15:14:37,744 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:37,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-12-04T15:14:37,780 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:37,783 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0027e0d5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44e441a8 2024-12-04T15:14:37,830 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:37,831 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-04T15:14:37,980 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:37,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:37,993 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:38,011 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-04T15:14:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:14:38,021 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:38,023 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e18b0bf to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3394cc1f 2024-12-04T15:14:38,067 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x2e18b0bf {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e18b0bf to 127.0.0.1:64157 2024-12-04T15:14:38,067 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:38,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:14:38,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-04T15:14:38,267 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:38,274 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:38,274 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:38,275 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:38,275 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:38,277 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10cede1b to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e0894f8 2024-12-04T15:14:38,278 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:38,334 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:14:38,337 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:38,338 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-04T15:14:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-04T15:14:38,344 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:38,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T15:14:38,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 252 msec 2024-12-04T15:14:38,356 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-04T15:14:38,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 345 msec 2024-12-04T15:14:38,375 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:38,378 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2268053e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b9adc06 2024-12-04T15:14:38,419 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:38,419 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-04T15:14:38,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:38,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:38,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:38,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T15:14:38,681 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:38,683 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b5dd0f1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62999643 2024-12-04T15:14:38,707 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0b5dd0f1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b5dd0f1 to 127.0.0.1:64157 2024-12-04T15:14:38,708 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:38,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T15:14:38,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-04T15:14:38,896 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:38,902 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:38,902 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:38,903 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:38,903 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:38,906 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49f6b056 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75cb48b6 2024-12-04T15:14:38,911 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:38,953 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:38,954 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:38,957 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-04T15:14:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-04T15:14:38,965 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:38,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T15:14:38,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 228 msec 2024-12-04T15:14:38,975 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:38,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 300 msec 2024-12-04T15:14:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T15:14:38,995 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:38,997 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:39,007 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1af18447 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bfba28a 2024-12-04T15:14:39,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:39,013 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:39,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:39,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:39,033 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=21 2024-12-04T15:14:39,033 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T15:14:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T15:14:39,039 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:39,039 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:39,048 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:39,049 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:39,051 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:39,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:39,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T15:14:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T15:14:39,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23 2024-12-04T15:14:39,241 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:39,241 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:39,241 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:39,242 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:39,242 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:39,244 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09143c83 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7cad9832 2024-12-04T15:14:39,246 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:39,247 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:39,264 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-12-04T15:14:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=23 2024-12-04T15:14:39,277 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:39,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=21 2024-12-04T15:14:39,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-12-04T15:14:39,290 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:39,292 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:39,312 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:39,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 299 msec 2024-12-04T15:14:39,321 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:39,329 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=22, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:39,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=22, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=305 msec 2024-12-04T15:14:39,335 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-04T15:14:39,358 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T15:14:39,360 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:39,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-04T15:14:39,370 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:39,378 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:39,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=17 msec 2024-12-04T15:14:39,380 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-04T15:14:39,477 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., hostname=c6f2e1d35cf1,37535,1733325265751, seqNum=2] 2024-12-04T15:14:39,595 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=266 (was 214) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x388552d5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1af18447-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b152d45-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x648d6c1e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b152d45 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3cee4cc5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x49f6b056-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x09143c83-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3cee4cc5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3a53f3f1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x648d6c1e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6696e4e2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2268053e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb-prefix:c6f2e1d35cf1,37535,1733325265751.rep java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x10cede1b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6696e4e2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x10cede1b-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2268053e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1780837858_22 at /127.0.0.1:41032 [Receiving block BP-1046799081-172.17.0.3-1733325260798:blk_1073741837_1013] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0027e0d5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x09143c83 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x49f6b056 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0027e0d5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x648d6c1e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3a53f3f1-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x370af7b2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x10cede1b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6696e4e2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x388552d5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x370af7b2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b152d45-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1af18447 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/c6f2e1d35cf1:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.unpark(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:177) java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.tryFire(CompletableFuture.java:1848) java.base@17.0.11/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) java.base@17.0.11/java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:614) java.base@17.0.11/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:844) java.base@17.0.11/java.util.concurrent.CompletableFuture$Completion.exec(CompletableFuture.java:483) java.base@17.0.11/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373) java.base@17.0.11/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1182) java.base@17.0.11/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1655) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1622) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x370af7b2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1046799081-172.17.0.3-1733325260798:blk_1073741837_1013, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3a53f3f1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x56b58936-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x49f6b056-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0027e0d5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1af18447-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x56b58936-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2268053e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x388552d5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3cee4cc5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x09143c83-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x56b58936 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=507 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1009 (was 932) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3012 (was 3220) 2024-12-04T15:14:39,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=266, OpenFileDescriptor=507, MaxFileDescriptor=1048576, SystemLoadAverage=1009, ProcessCount=11, AvailableMemoryMB=3011 2024-12-04T15:14:39,609 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:39,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:14:39,617 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:39,618 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ae767e5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d084d45 2024-12-04T15:14:39,639 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x3ae767e5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3ae767e5 to 127.0.0.1:64157 2024-12-04T15:14:39,639 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:14:39,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:39,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26 2024-12-04T15:14:39,902 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:14:39,981 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:39,988 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-04T15:14:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-04T15:14:39,995 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:40,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T15:14:40,002 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:40,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 256 msec 2024-12-04T15:14:40,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 393 msec 2024-12-04T15:14:40,027 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:40,029 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ed7300f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18267c6d 2024-12-04T15:14:40,078 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:40,079 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:40,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T15:14:40,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:40,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:40,255 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:40,257 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fcae16e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bdde57b 2024-12-04T15:14:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T15:14:40,351 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1fcae16e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fcae16e to 127.0.0.1:64157 2024-12-04T15:14:40,351 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T15:14:40,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:40,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-04T15:14:40,560 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:40,567 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:40,567 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:40,568 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:40,568 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:40,570 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45a162c8 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cb75fc5 2024-12-04T15:14:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T15:14:40,582 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:40,632 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:40,636 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:40,641 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-04T15:14:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-04T15:14:40,659 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:40,676 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:40,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 428 msec 2024-12-04T15:14:40,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-04T15:14:40,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 265 msec 2024-12-04T15:14:40,699 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:40,701 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21325222 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52913cb7 2024-12-04T15:14:40,748 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:40,748 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-04T15:14:40,901 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:40,902 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:14:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:40,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:14:40,914 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:40,915 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08de6a16 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72303fb2 2024-12-04T15:14:40,971 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x08de6a16 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08de6a16 to 127.0.0.1:64157 2024-12-04T15:14:40,971 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:40,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:14:41,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-04T15:14:41,159 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:41,172 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:41,172 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:41,174 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:41,175 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,178 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5666ed3a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24814baf 2024-12-04T15:14:41,178 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:14:41,234 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:41,235 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:41,241 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-04T15:14:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-04T15:14:41,247 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:41,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T15:14:41,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 249 msec 2024-12-04T15:14:41,253 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:14:41,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 351 msec 2024-12-04T15:14:41,273 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,284 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4abdb907 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5479063c 2024-12-04T15:14:41,330 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:41,331 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-04T15:14:41,536 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:41,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-04T15:14:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T15:14:41,558 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,560 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60111a1c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fa81a29 2024-12-04T15:14:41,602 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x60111a1c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60111a1c to 127.0.0.1:64157 2024-12-04T15:14:41,602 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:41,603 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:41,605 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:41,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=63 msec 2024-12-04T15:14:41,615 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T15:14:41,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-04T15:14:41,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:14:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:41,677 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:14:41,678 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a68c556 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e76844a 2024-12-04T15:14:41,717 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5a68c556 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5a68c556 to 127.0.0.1:64157 2024-12-04T15:14:41,717 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:41,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:14:41,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33 2024-12-04T15:14:41,911 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:41,917 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:41,917 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:41,917 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:41,918 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,920 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78a2553c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@650dd916 2024-12-04T15:14:41,923 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:41,955 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:41,955 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:41,959 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-04T15:14:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-04T15:14:41,982 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:41,991 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:41,992 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17ddf1e4 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70366a92 2024-12-04T15:14:41,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T15:14:41,997 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:14:41,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 243 msec 2024-12-04T15:14:42,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 325 msec 2024-12-04T15:14:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-04T15:14:42,020 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:42,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:14:42,026 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:42,027 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:14:42,037 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:42,039 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x586dd140 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9d7e710 2024-12-04T15:14:42,095 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x586dd140 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x586dd140 to 127.0.0.1:64157 2024-12-04T15:14:42,095 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:42,096 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:42,099 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:42,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=78 msec 2024-12-04T15:14:42,104 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-04T15:14:42,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-04T15:14:42,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-04T15:14:42,188 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:42,189 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:42,189 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:42,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:42,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-04T15:14:42,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36 2024-12-04T15:14:42,381 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:42,381 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:42,381 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:42,381 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:42,382 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:42,384 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x466153ae to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d3d430a 2024-12-04T15:14:42,387 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:42,388 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:42,393 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36 2024-12-04T15:14:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=36 2024-12-04T15:14:42,408 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:42,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-04T15:14:42,429 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:42,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-04T15:14:42,429 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:42,455 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:42,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 294 msec 2024-12-04T15:14:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-04T15:14:42,501 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:42,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:42,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:42,506 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-04T15:14:42,509 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:42,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-04T15:14:42,511 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-04T15:14:42,616 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:42,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T15:14:42,625 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:42,632 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:42,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-12-04T15:14:42,636 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-04T15:14:42,736 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:42,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=290 (was 266) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21325222-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5666ed3a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x466153ae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x17ddf1e4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21325222-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x466153ae-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x17ddf1e4-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x78a2553c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5666ed3a-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x78a2553c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ed7300f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4abdb907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5666ed3a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x45a162c8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ed7300f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4abdb907-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x45a162c8-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4abdb907-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x17ddf1e4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x466153ae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x45a162c8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21325222 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x78a2553c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ed7300f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=540 (was 507) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1097 (was 1009) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2947 (was 3011) 2024-12-04T15:14:42,773 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=290, OpenFileDescriptor=540, MaxFileDescriptor=1048576, SystemLoadAverage=1097, ProcessCount=11, AvailableMemoryMB=2947 2024-12-04T15:14:42,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:14:42,785 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:42,787 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39411dcb to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@396d5e5a 2024-12-04T15:14:42,804 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x39411dcb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x39411dcb to 127.0.0.1:64157 2024-12-04T15:14:42,804 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:42,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:14:43,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40 2024-12-04T15:14:43,001 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:43,064 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:43,070 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-04T15:14:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-04T15:14:43,080 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:43,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-04T15:14:43,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 241 msec 2024-12-04T15:14:43,089 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:14:43,096 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:43,100 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a5f3ad4 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b2dc422 2024-12-04T15:14:43,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 314 msec 2024-12-04T15:14:43,127 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:43,127 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-04T15:14:43,405 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:43,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-04T15:14:43,413 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:43,417 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05bbcb37 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f4bf505 2024-12-04T15:14:43,436 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x05bbcb37 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05bbcb37 to 127.0.0.1:64157 2024-12-04T15:14:43,436 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:43,436 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:43,438 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:43,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=31 msec 2024-12-04T15:14:43,440 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-04T15:14:43,526 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-04T15:14:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:43,541 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T15:14:43,547 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:43,555 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:43,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-12-04T15:14:43,565 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-04T15:14:43,655 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:43,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-04T15:14:43,664 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:14:43,665 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5188475e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1019b943 2024-12-04T15:14:43,692 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5188475e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5188475e to 127.0.0.1:64157 2024-12-04T15:14:43,692 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:43,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-04T15:14:43,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44 2024-12-04T15:14:43,913 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-04T15:14:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-04T15:14:44,028 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 2, currentBandwidth=0 2024-12-04T15:14:44,044 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-12-04T15:14:44,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=44 2024-12-04T15:14:44,051 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:44,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-04T15:14:44,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 303 msec 2024-12-04T15:14:44,060 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:44,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 401 msec 2024-12-04T15:14:44,103 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:14:44,106 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dd5363c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@359434a3 2024-12-04T15:14:44,131 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:44,133 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06fcc4df to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7733033 2024-12-04T15:14:44,158 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:44,159 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:44,160 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:44,160 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-04T15:14:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-04T15:14:44,295 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-04T15:14:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:44,299 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-04T15:14:44,312 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:44,314 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:44,321 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:44,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:44,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:44,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:44,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:44,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:44,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-04T15:14:44,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-04T15:14:44,514 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:44,514 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:44,514 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:44,514 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:44,514 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:44,516 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a86f4e7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26b6f5e5 2024-12-04T15:14:44,517 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:44,518 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:44,521 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-04T15:14:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=46 2024-12-04T15:14:44,525 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:44,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-12-04T15:14:44,530 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:44,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec 2024-12-04T15:14:44,530 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:44,540 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:44,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 241 msec 2024-12-04T15:14:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-04T15:14:44,625 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:44,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:14:44,634 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-04T15:14:44,635 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-04T15:14:44,635 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-04T15:14:44,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-04T15:14:44,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-04T15:14:44,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-04T15:14:44,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-04T15:14:44,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-04T15:14:44,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:14:44,752 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:14:44,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-04T15:14:44,801 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-04T15:14:44,801 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-04T15:14:44,801 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:44,801 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-04T15:14:44,802 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:14:44,803 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x14cf675b to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@505c07f5 2024-12-04T15:14:44,804 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:44,804 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:44,808 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-04T15:14:44,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-04T15:14:44,814 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:44,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-04T15:14:44,818 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-04T15:14:44,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 167 msec 2024-12-04T15:14:44,818 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-04T15:14:44,827 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-04T15:14:44,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 200 msec 2024-12-04T15:14:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-04T15:14:44,946 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-04T15:14:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:44,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:44,953 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-04T15:14:44,955 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:44,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:14:44,957 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-04T15:14:45,066 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:45,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:14:45,084 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:45,086 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=20 msec 2024-12-04T15:14:45,089 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-04T15:14:45,186 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:45,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=302 (was 290) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x06fcc4df-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd5363c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5a86f4e7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0a5f3ad4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0a5f3ad4-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0a5f3ad4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd5363c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14cf675b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5a86f4e7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14cf675b-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x06fcc4df java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd5363c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14cf675b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5a86f4e7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x06fcc4df-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=555 (was 540) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1089 (was 1097), ProcessCount=11 (was 11), AvailableMemoryMB=2909 (was 2947) 2024-12-04T15:14:45,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=1089, ProcessCount=11, AvailableMemoryMB=2907 2024-12-04T15:14:45,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-04T15:14:45,247 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-04T15:14:45,258 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,268 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=22 msec 2024-12-04T15:14:45,268 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-04T15:14:45,355 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-04T15:14:45,362 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:14:45,374 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:45,380 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=18 msec 2024-12-04T15:14:45,387 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-04T15:14:45,476 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:45,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-04T15:14:45,489 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:45,495 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=19 msec 2024-12-04T15:14:45,503 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-04T15:14:45,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:45,624 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302 (was 302), OpenFileDescriptor=553 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1089 (was 1089), ProcessCount=11 (was 11), AvailableMemoryMB=2894 (was 2907) 2024-12-04T15:14:45,642 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302, OpenFileDescriptor=553, MaxFileDescriptor=1048576, SystemLoadAverage=1089, ProcessCount=11, AvailableMemoryMB=2892 2024-12-04T15:14:45,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:14:45,656 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-04T15:14:45,659 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=15 msec 2024-12-04T15:14:45,661 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-04T15:14:45,760 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-04T15:14:45,763 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:45,769 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T15:14:45,776 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-12-04T15:14:45,780 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-04T15:14:45,886 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:45,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:14:45,891 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:45,893 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:45,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-04T15:14:45,895 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-04T15:14:45,995 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:46,017 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302 (was 302), OpenFileDescriptor=553 (was 553), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1089 (was 1089), ProcessCount=11 (was 11), AvailableMemoryMB=2884 (was 2892) 2024-12-04T15:14:46,027 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=302, OpenFileDescriptor=553, MaxFileDescriptor=1048576, SystemLoadAverage=1089, ProcessCount=11, AvailableMemoryMB=2884 2024-12-04T15:14:46,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:46,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T15:14:46,031 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,032 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x521554ac to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c086fb8 2024-12-04T15:14:46,066 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x521554ac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x521554ac to 127.0.0.1:64157 2024-12-04T15:14:46,066 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:46,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T15:14:46,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58 2024-12-04T15:14:46,248 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:46,278 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:46,281 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-12-04T15:14:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-12-04T15:14:46,284 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:46,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-04T15:14:46,288 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:46,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-04T15:14:46,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 260 msec 2024-12-04T15:14:46,301 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,303 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a05ef1c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f17cd55 2024-12-04T15:14:46,312 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:46,312 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-04T15:14:46,345 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:46,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T15:14:46,349 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,349 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x357dadf5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39fa7299 2024-12-04T15:14:46,358 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x357dadf5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x357dadf5 to 127.0.0.1:64157 2024-12-04T15:14:46,358 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:46,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T15:14:46,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-04T15:14:46,530 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:46,532 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:46,532 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:46,533 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:46,533 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,535 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x252c9da8 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b58a41f 2024-12-04T15:14:46,536 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:46,554 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:46,554 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:46,554 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-04T15:14:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-12-04T15:14:46,557 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:46,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-04T15:14:46,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-04T15:14:46,560 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:46,562 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec 2024-12-04T15:14:46,575 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,576 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x528708c2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e855fdc 2024-12-04T15:14:46,587 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:46,588 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-04T15:14:46,666 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:46,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-04T15:14:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T15:14:46,672 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,674 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e7cad19 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28dfa846 2024-12-04T15:14:46,683 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5e7cad19 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e7cad19 to 127.0.0.1:64157 2024-12-04T15:14:46,683 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:46,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T15:14:46,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-04T15:14:46,855 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:46,856 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:46,856 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:46,856 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:46,857 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,859 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5196730e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fe33149 2024-12-04T15:14:46,859 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:46,877 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:46,877 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:46,877 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-04T15:14:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-04T15:14:46,881 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:46,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-04T15:14:46,884 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-04T15:14:46,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-04T15:14:46,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec 2024-12-04T15:14:46,893 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,895 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f6ea0e3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b509820 2024-12-04T15:14:46,904 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:46,904 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-04T15:14:46,985 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:46,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:46,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:14:46,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T15:14:46,991 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:46,992 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1eddac07 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f9c346c 2024-12-04T15:14:47,004 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1eddac07 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1eddac07 to 127.0.0.1:64157 2024-12-04T15:14:47,004 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:47,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T15:14:47,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-04T15:14:47,176 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:47,177 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:47,177 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:47,177 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:47,177 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:47,178 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48efa64e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7353b9d0 2024-12-04T15:14:47,179 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:47,202 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:47,202 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:47,202 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-04T15:14:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=64 2024-12-04T15:14:47,207 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:47,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-04T15:14:47,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-04T15:14:47,209 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:14:47,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 223 msec 2024-12-04T15:14:47,217 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:47,218 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7166eb48 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e16bd3b 2024-12-04T15:14:47,229 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:47,229 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-04T15:14:47,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:47,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:47,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-04T15:14:47,312 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:47,313 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:47,313 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:47,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:47,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-04T15:14:47,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-04T15:14:47,481 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:47,482 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:47,482 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:47,482 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:47,482 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:47,487 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5fce53cf to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c3755bd 2024-12-04T15:14:47,488 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:47,488 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:47,492 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-04T15:14:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-04T15:14:47,495 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:47,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-04T15:14:47,498 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:47,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec 2024-12-04T15:14:47,498 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:47,505 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:47,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 198 msec 2024-12-04T15:14:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-04T15:14:47,625 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:47,626 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:14:47,629 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:47,631 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:47,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:14:47,633 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-04T15:14:47,736 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:47,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:14:47,740 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:47,741 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:47,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:14:47,743 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-04T15:14:47,845 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:47,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=326 (was 302) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5f6ea0e3-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4a05ef1c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x252c9da8-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48efa64e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5196730e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x252c9da8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5196730e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5196730e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5f6ea0e3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5f6ea0e3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x528708c2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5fce53cf-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x528708c2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7166eb48-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7166eb48 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x528708c2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48efa64e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4a05ef1c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x252c9da8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48efa64e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5fce53cf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7166eb48-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5fce53cf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4a05ef1c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=585 (was 553) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1089 (was 1089), ProcessCount=11 (was 11), AvailableMemoryMB=2880 (was 2884) 2024-12-04T15:14:47,875 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=326, OpenFileDescriptor=585, MaxFileDescriptor=1048576, SystemLoadAverage=1089, ProcessCount=11, AvailableMemoryMB=2880 2024-12-04T15:14:47,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-04T15:14:47,880 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:47,881 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0fef5072 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8ae5f7f 2024-12-04T15:14:47,891 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0fef5072 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0fef5072 to 127.0.0.1:64157 2024-12-04T15:14:47,891 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:47,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-04T15:14:48,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70 2024-12-04T15:14:48,071 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:48,103 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:48,106 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70 2024-12-04T15:14:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=70 2024-12-04T15:14:48,109 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:48,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-04T15:14:48,112 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:48,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-04T15:14:48,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 235 msec 2024-12-04T15:14:48,121 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,123 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ea897f5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a8f8768 2024-12-04T15:14:48,155 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:48,155 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-04T15:14:48,196 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:48,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-04T15:14:48,202 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,203 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f42cc2a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d30d3af 2024-12-04T15:14:48,221 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x7f42cc2a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7f42cc2a to 127.0.0.1:64157 2024-12-04T15:14:48,221 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:48,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-04T15:14:48,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-04T15:14:48,396 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:48,397 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:48,397 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:48,397 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:48,398 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,400 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x019a67c9 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6970973b 2024-12-04T15:14:48,400 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:48,422 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:48,422 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:48,423 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-04T15:14:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=72 2024-12-04T15:14:48,426 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:48,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-12-04T15:14:48,428 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:14:48,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-04T15:14:48,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 231 msec 2024-12-04T15:14:48,438 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,440 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x609afee7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d4be58 2024-12-04T15:14:48,454 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:48,454 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-04T15:14:48,515 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:48,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:48,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:14:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-04T15:14:48,523 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,525 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53930504 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f19cad 2024-12-04T15:14:48,541 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x53930504 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x53930504 to 127.0.0.1:64157 2024-12-04T15:14:48,541 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:48,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-04T15:14:48,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-04T15:14:48,712 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:48,713 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:48,713 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:48,713 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:48,713 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,715 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x114f71ae to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f06760c 2024-12-04T15:14:48,715 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:48,734 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:48,734 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:48,734 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-04T15:14:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=74 2024-12-04T15:14:48,737 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:48,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-04T15:14:48,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-04T15:14:48,740 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:14:48,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 222 msec 2024-12-04T15:14:48,749 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,750 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57a18bac to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126c1c7 2024-12-04T15:14:48,762 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:48,762 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:48,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-04T15:14:48,835 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:48,835 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:48,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:48,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:14:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:48,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-04T15:14:48,842 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:48,843 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e2e2cc4 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@525916c0 2024-12-04T15:14:48,858 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x2e2e2cc4 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e2e2cc4 to 127.0.0.1:64157 2024-12-04T15:14:48,859 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:48,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-04T15:14:49,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-04T15:14:49,030 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:49,031 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:49,031 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:49,031 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:49,031 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,033 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59b55eed to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55e5dd34 2024-12-04T15:14:49,033 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:49,050 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:49,051 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:49,051 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-04T15:14:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=76 2024-12-04T15:14:49,055 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:49,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-04T15:14:49,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-04T15:14:49,057 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:14:49,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 219 msec 2024-12-04T15:14:49,066 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,067 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21cace42 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@286f434d 2024-12-04T15:14:49,103 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:49,103 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-04T15:14:49,155 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:49,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:14:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-04T15:14:49,161 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,162 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a08bfa3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@519d00b4 2024-12-04T15:14:49,178 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x4a08bfa3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4a08bfa3 to 127.0.0.1:64157 2024-12-04T15:14:49,178 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:49,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-04T15:14:49,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-04T15:14:49,349 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:49,352 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:49,352 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:49,352 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:49,352 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,355 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3884b1bf to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ae490c5 2024-12-04T15:14:49,356 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:49,375 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:49,376 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:49,376 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-04T15:14:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=78 2024-12-04T15:14:49,379 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:49,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-04T15:14:49,381 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:14:49,381 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-04T15:14:49,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 224 msec 2024-12-04T15:14:49,392 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,393 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70337400 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a46dafe 2024-12-04T15:14:49,408 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:49,408 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-04T15:14:49,475 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:49,476 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-04T15:14:49,481 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,482 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5d65ef84 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@19f2441a 2024-12-04T15:14:49,491 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5d65ef84 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5d65ef84 to 127.0.0.1:64157 2024-12-04T15:14:49,491 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:49,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-04T15:14:49,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-04T15:14:49,662 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:49,664 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:49,664 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:49,664 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:49,665 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,666 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x14b85e79 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@66caee2f 2024-12-04T15:14:49,666 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:49,689 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:49,689 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:49,689 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-04T15:14:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=80 2024-12-04T15:14:49,692 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:49,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-04T15:14:49,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-04T15:14:49,694 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:49,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec 2024-12-04T15:14:49,703 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,704 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4461c193 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d2b8d90 2024-12-04T15:14:49,741 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:49,742 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-04T15:14:49,795 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:49,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:49,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:14:49,801 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:49,802 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76c988d1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24322cdc 2024-12-04T15:14:49,816 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x76c988d1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76c988d1 to 127.0.0.1:64157 2024-12-04T15:14:49,816 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:49,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:14:50,020 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:14:50,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-04T15:14:50,047 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:50,050 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:50,050 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:50,050 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:50,051 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:50,052 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3caeea52 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ec7a72b 2024-12-04T15:14:50,053 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:50,074 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:50,074 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:50,075 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-04T15:14:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-12-04T15:14:50,078 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:50,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-04T15:14:50,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-04T15:14:50,080 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:50,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 282 msec 2024-12-04T15:14:50,090 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:50,092 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x516ec541 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c7f409f 2024-12-04T15:14:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-04T15:14:50,115 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:50,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:50,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-04T15:14:50,121 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:50,122 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32382878 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e3bc6fd 2024-12-04T15:14:50,180 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x32382878 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32382878 to 127.0.0.1:64157 2024-12-04T15:14:50,180 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:50,180 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:50,180 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-04T15:14:50,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-04T15:14:50,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-04T15:14:50,463 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:50,464 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-04T15:14:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-12-04T15:14:50,467 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:50,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-04T15:14:50,469 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:14:50,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-04T15:14:50,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 351 msec 2024-12-04T15:14:50,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-04T15:14:50,745 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:50,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:50,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:50,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-04T15:14:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:50,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T15:14:50,752 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:50,753 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55cf26f1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7adb3180 2024-12-04T15:14:50,796 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x55cf26f1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x55cf26f1 to 127.0.0.1:64157 2024-12-04T15:14:50,796 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T15:14:50,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T15:14:51,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-04T15:14:51,092 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:51,093 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:51,093 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:51,094 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:51,094 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,096 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7cd1bed2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13bbcfe9 2024-12-04T15:14:51,096 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:51,111 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:51,112 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:51,112 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-04T15:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-04T15:14:51,114 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:51,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-04T15:14:51,116 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-04T15:14:51,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:14:51,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 368 msec 2024-12-04T15:14:51,127 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,128 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ca1df06 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@736b0814 2024-12-04T15:14:51,180 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:51,181 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-04T15:14:51,375 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:51,376 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:51,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-04T15:14:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T15:14:51,381 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,382 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6aa16b1a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18b03aa4 2024-12-04T15:14:51,391 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x6aa16b1a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6aa16b1a to 127.0.0.1:64157 2024-12-04T15:14:51,391 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:51,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T15:14:51,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-04T15:14:51,561 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:51,563 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:51,563 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:51,563 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:51,563 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,565 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5471a1c7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b2deeaf 2024-12-04T15:14:51,565 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:51,584 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:51,584 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:51,584 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-04T15:14:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-04T15:14:51,588 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:51,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-04T15:14:51,592 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-04T15:14:51,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-04T15:14:51,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 215 msec 2024-12-04T15:14:51,602 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,603 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03feb39e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53db9bfc 2024-12-04T15:14:51,616 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:51,616 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-04T15:14:51,695 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:51,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:51,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:14:51,702 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:51,703 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:51,703 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:51,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:51,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:51,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:51,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:51,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:51,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:14:51,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-04T15:14:51,866 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:51,866 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:51,866 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:51,867 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:51,867 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:51,868 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2bfcc6e0 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14a0881c 2024-12-04T15:14:51,868 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:51,869 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:51,871 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-04T15:14:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-04T15:14:51,874 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:51,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-04T15:14:51,875 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:51,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec 2024-12-04T15:14:51,875 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:51,883 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:51,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-04T15:14:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-04T15:14:52,015 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:52,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:14:52,018 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:52,019 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:52,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-04T15:14:52,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-04T15:14:52,127 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:52,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:52,131 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-04T15:14:52,133 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:52,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:14:52,134 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-04T15:14:52,235 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:52,256 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=380 (was 326) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7cd1bed2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x019a67c9-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x516ec541 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2bfcc6e0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x59b55eed-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4461c193 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x516ec541-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x114f71ae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70337400-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x57a18bac java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21cace42-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x59b55eed java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x019a67c9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70337400-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7cd1bed2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21cace42-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x609afee7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x019a67c9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0ea897f5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x114f71ae-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x114f71ae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14b85e79 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x609afee7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70337400 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x03feb39e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5ca1df06-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5ca1df06-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14b85e79-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7cd1bed2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3caeea52 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2bfcc6e0-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x14b85e79-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2bfcc6e0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3884b1bf-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3caeea52-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0ea897f5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0ea897f5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3884b1bf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5471a1c7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4461c193-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5ca1df06 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x59b55eed-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5471a1c7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3884b1bf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x57a18bac-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x57a18bac-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x03feb39e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3caeea52-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x609afee7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4461c193-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5471a1c7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21cace42 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x516ec541-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x03feb39e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=657 (was 585) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1098 (was 1089) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2712 (was 2880) 2024-12-04T15:14:52,269 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=380, OpenFileDescriptor=657, MaxFileDescriptor=1048576, SystemLoadAverage=1098, ProcessCount=11, AvailableMemoryMB=2711 2024-12-04T15:14:52,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:14:52,274 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:52,276 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49fed5ca to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f1590df 2024-12-04T15:14:52,287 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x49fed5ca {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x49fed5ca to 127.0.0.1:64157 2024-12-04T15:14:52,287 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:52,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:14:52,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94 2024-12-04T15:14:52,470 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:52,506 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:52,509 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-04T15:14:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-04T15:14:52,512 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:52,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-04T15:14:52,514 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:52,514 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-04T15:14:52,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 244 msec 2024-12-04T15:14:52,526 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:52,527 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1c48f8a7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d940f83 2024-12-04T15:14:52,541 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:52,541 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-04T15:14:52,595 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:52,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-04T15:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-04T15:14:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:14:52,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:14:52,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-04T15:14:52,765 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-04T15:14:52,766 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:14:52,766 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:14:52,767 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:52,767 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:52,768 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x761b54b5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6757e5c3 2024-12-04T15:14:52,769 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:52,786 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:52,786 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:14:52,787 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-04T15:14:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-04T15:14:52,789 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:52,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-04T15:14:52,791 INFO [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-04T15:14:52,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:14:52,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 191 msec 2024-12-04T15:14:52,801 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:52,802 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a82b7a5 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f9badf6 2024-12-04T15:14:52,812 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:52,812 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:52,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-04T15:14:52,925 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:52,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:52,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:14:52,932 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:52,933 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:52,933 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:52,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:52,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:52,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:52,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:52,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:14:53,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-04T15:14:53,099 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:53,099 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:53,099 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:53,099 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:53,100 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:53,101 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38f3e0df to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@409e3af6 2024-12-04T15:14:53,101 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:53,102 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:53,104 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-04T15:14:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-04T15:14:53,108 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:53,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-04T15:14:53,111 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:53,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-04T15:14:53,111 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:53,119 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:53,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 192 msec 2024-12-04T15:14:53,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-04T15:14:53,245 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:53,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:14:53,247 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:53,249 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:53,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:14:53,250 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-04T15:14:53,356 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:53,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-04T15:14:53,359 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:53,360 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:53,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:14:53,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:53,449 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:14:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-04T15:14:53,466 INFO [Time-limited test {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:53,486 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=392 (was 380) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x38f3e0df-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x761b54b5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1c48f8a7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x761b54b5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1c48f8a7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x38f3e0df java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2a82b7a5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x761b54b5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2a82b7a5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1c48f8a7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x38f3e0df-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2a82b7a5-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=673 (was 657) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1098 (was 1098), ProcessCount=11 (was 11), AvailableMemoryMB=2683 (was 2711) 2024-12-04T15:14:53,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=392, OpenFileDescriptor=673, MaxFileDescriptor=1048576, SystemLoadAverage=1098, ProcessCount=11, AvailableMemoryMB=2683 2024-12-04T15:14:53,499 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-04T15:14:53,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:53,502 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:53,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:14:53,503 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:53,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:14:53,505 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:53,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-04T15:14:53,605 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-04T15:14:53,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-04T15:14:53,608 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:53,608 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:53,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-04T15:14:53,610 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-04T15:14:53,715 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:53,716 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:14:53,719 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:53,720 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:53,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:14:53,722 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-04T15:14:53,825 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:53,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=394 (was 392) - Thread LEAK? -, OpenFileDescriptor=675 (was 673) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1098 (was 1098), ProcessCount=11 (was 11), AvailableMemoryMB=2672 (was 2683) 2024-12-04T15:14:53,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=394, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=1098, ProcessCount=11, AvailableMemoryMB=2669 2024-12-04T15:14:53,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:53,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-04T15:14:53,908 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:53,909 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6584626c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7923e5c8 2024-12-04T15:14:53,945 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x6584626c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6584626c to 127.0.0.1:64157 2024-12-04T15:14:53,945 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:53,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-04T15:14:54,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105 2024-12-04T15:14:54,126 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:54,182 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:54,194 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-12-04T15:14:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=105 2024-12-04T15:14:54,198 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:54,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-04T15:14:54,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 227 msec 2024-12-04T15:14:54,203 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:54,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 299 msec 2024-12-04T15:14:54,217 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:54,219 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x329a782b to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42ce71d3 2024-12-04T15:14:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-04T15:14:54,225 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:14:54,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-04T15:14:54,233 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:54,234 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:54,234 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:54,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:54,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:54,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:54,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:54,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:54,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:54,260 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:54,260 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-04T15:14:54,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-04T15:14:54,408 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:54,408 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:54,409 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:54,409 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:54,409 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:54,410 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x18db7476 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@227f824e 2024-12-04T15:14:54,410 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:54,411 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:54,413 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-04T15:14:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=107 2024-12-04T15:14:54,416 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:54,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-04T15:14:54,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-04T15:14:54,419 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:54,419 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:54,436 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:54,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 209 msec 2024-12-04T15:14:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-04T15:14:54,545 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:54,545 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:54,548 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-04T15:14:54,550 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:54,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:14:54,558 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-04T15:14:54,656 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:54,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:54,661 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T15:14:54,666 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:54,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=11 msec 2024-12-04T15:14:54,668 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-04T15:14:54,776 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:54,817 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=399 (was 394) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x18db7476-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x18db7476 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x329a782b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x18db7476-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x329a782b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x329a782b-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=683 (was 677) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1098 (was 1098), ProcessCount=11 (was 11), AvailableMemoryMB=2657 (was 2669) 2024-12-04T15:14:54,855 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=399, OpenFileDescriptor=683, MaxFileDescriptor=1048576, SystemLoadAverage=1098, ProcessCount=11, AvailableMemoryMB=2656 2024-12-04T15:14:54,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 319 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:52758 deadline: 1733325354887, exception=java.io.IOException: Replication peer modification disabled 2024-12-04T15:14:54,913 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 26 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T15:14:55,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 320 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:52758 deadline: 1733325355027, exception=java.io.IOException: Replication peer modification disabled 2024-12-04T15:14:55,028 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 144 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T15:14:55,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-04T15:14:54.915Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:14:55.029Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:14:55,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:14:55,037 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:55,038 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:55,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:14:55,039 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-04T15:14:55,146 INFO [Time-limited test {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:55,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:55,151 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:55,152 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:55,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-04T15:14:55,154 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-04T15:14:55,165 INFO [Time-limited test {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:55,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=399 (was 399), OpenFileDescriptor=683 (was 683), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1082 (was 1098), ProcessCount=11 (was 11), AvailableMemoryMB=2640 (was 2656) 2024-12-04T15:14:55,215 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=399, OpenFileDescriptor=683, MaxFileDescriptor=1048576, SystemLoadAverage=1082, ProcessCount=11, AvailableMemoryMB=2639 2024-12-04T15:14:55,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:55,220 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T15:14:55,222 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0585a823 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4da1175b 2024-12-04T15:14:55,238 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0585a823 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0585a823 to 127.0.0.1:64157 2024-12-04T15:14:55,238 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:55,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T15:14:55,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113 2024-12-04T15:14:55,472 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T15:14:55,549 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:55,574 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-12-04T15:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=113 2024-12-04T15:14:55,581 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:55,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-04T15:14:55,591 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:55,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 270 msec 2024-12-04T15:14:55,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 375 msec 2024-12-04T15:14:55,619 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:55,636 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7954a9be to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a94b205 2024-12-04T15:14:55,803 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:55,804 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-04T15:14:55,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:55,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:55,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-04T15:14:55,864 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:55,864 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:55,865 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:55,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:55,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-04T15:14:56,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-04T15:14:56,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:56,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:56,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:56,044 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:56,044 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:56,046 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x483b134a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10601051 2024-12-04T15:14:56,046 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:56,047 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:56,052 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-04T15:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=115 2024-12-04T15:14:56,057 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:56,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-04T15:14:56,062 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:56,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 170 msec 2024-12-04T15:14:56,062 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:56,074 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:56,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 216 msec 2024-12-04T15:14:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-04T15:14:56,176 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:56,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:56,185 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-04T15:14:56,188 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:56,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=11 msec 2024-12-04T15:14:56,190 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-04T15:14:56,296 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:56,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=405 (was 399) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7954a9be-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x483b134a-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7954a9be-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7954a9be java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x483b134a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x483b134a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=691 (was 683) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1082 (was 1082), ProcessCount=11 (was 11), AvailableMemoryMB=2611 (was 2639) 2024-12-04T15:14:56,347 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=405, OpenFileDescriptor=691, MaxFileDescriptor=1048576, SystemLoadAverage=1082, ProcessCount=11, AvailableMemoryMB=2611 2024-12-04T15:14:56,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:14:56,352 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:56,353 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d43d703 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3173f1f5 2024-12-04T15:14:56,371 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x3d43d703 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d43d703 to 127.0.0.1:64157 2024-12-04T15:14:56,371 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:56,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:14:56,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118 2024-12-04T15:14:56,573 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:56,640 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:56,662 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-04T15:14:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-04T15:14:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:14:56,688 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:56,693 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:56,696 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52892c2f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76d6435b 2024-12-04T15:14:56,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-04T15:14:56,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-04T15:14:56,715 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:56,731 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:14:56,732 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:14:56,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 368 msec 2024-12-04T15:14:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-04T15:14:56,983 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:56,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:56,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-04T15:14:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:14:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:14:57,009 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:57,015 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a4a6ca3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f21b8bb 2024-12-04T15:14:57,079 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0a4a6ca3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a4a6ca3 to 127.0.0.1:64157 2024-12-04T15:14:57,079 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:14:57,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:14:57,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-04T15:14:57,280 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:14:57,287 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-04T15:14:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-04T15:14:57,294 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:57,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-04T15:14:57,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-04T15:14:57,298 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-04T15:14:57,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 303 msec 2024-12-04T15:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-04T15:14:57,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:14:57,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:14:57,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:14:57,360 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:57,361 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:57,365 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:57,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:14:57,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:57,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:57,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:57,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:57,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:57,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:14:57,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-04T15:14:57,548 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:57,548 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:57,548 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:57,549 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:14:57,549 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:14:57,551 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f36e4be to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32b2059b 2024-12-04T15:14:57,555 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:14:57,555 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:14:57,561 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-04T15:14:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-04T15:14:57,568 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:57,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-04T15:14:57,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-04T15:14:57,580 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:14:57,580 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:14:57,596 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:14:57,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 260 msec 2024-12-04T15:14:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-04T15:14:57,666 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:57,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:57,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:14:57,671 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:57,683 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:57,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=16 msec 2024-12-04T15:14:57,686 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-04T15:14:57,785 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:14:57,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:14:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:57,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-04T15:14:57,796 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:14:57,799 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:57,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-12-04T15:14:57,803 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-04T15:14:57,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:14:57,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=411 (was 405) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x52892c2f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0f36e4be java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0f36e4be-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0f36e4be-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x52892c2f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x52892c2f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=699 (was 691) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1082 (was 1082), ProcessCount=11 (was 11), AvailableMemoryMB=2545 (was 2611) 2024-12-04T15:14:57,967 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=411, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=1082, ProcessCount=11, AvailableMemoryMB=2544 2024-12-04T15:14:57,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:14:58,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:14:58,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126 2024-12-04T15:14:58,231 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:14:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:14:58,341 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:14:58,355 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-04T15:14:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-04T15:14:58,359 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-c6f2e1d35cf1,37535,1733325265751 (queues=1) is replicating from cluster=6aa3465c-d757-4950-ad0d-9868eb0e70b4 to cluster=6aa3465c-d757-4950-ad0d-9868eb0e70b4 2024-12-04T15:14:58,360 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=c6f2e1d35cf1%2C37535%2C1733325265751 2024-12-04T15:14:58,366 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:14:58,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-04T15:14:58,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 298 msec 2024-12-04T15:14:58,371 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:14:58,371 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-04T15:14:58,386 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: c6f2e1d35cf1%2C37535%2C1733325265751 2024-12-04T15:14:58,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 401 msec 2024-12-04T15:14:58,393 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.wal-reader.c6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.1733325271247, startPosition=0, beingWritten=true 2024-12-04T15:14:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-04T15:14:58,605 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:14:58,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:14:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:14:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:14:58,626 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-04T15:14:58,632 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:14:58,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=26 msec 2024-12-04T15:14:58,639 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:14:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-04T15:14:58,740 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-04T15:14:58,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:14:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:14:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:14:58,775 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:58,776 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:58,783 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:14:58,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:14:58,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:58,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:14:58,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:14:58,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:14:58,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:14:59,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129 2024-12-04T15:14:59,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:14:59,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:14:59,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:14:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:14:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:14:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:15:00,044 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.wal-reader.c6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-04T15:15:00,044 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-04T15:15:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:15:01,046 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 terminated 2024-12-04T15:15:01,047 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:01,057 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-12-04T15:15:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=129 2024-12-04T15:15:01,065 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:01,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-04T15:15:01,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1860 sec 2024-12-04T15:15:01,076 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:01,076 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:01,109 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:01,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.3660 sec 2024-12-04T15:15:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-04T15:15:02,934 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:02,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:02,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-04T15:15:02,942 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:02,947 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:02,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=13 msec 2024-12-04T15:15:02,953 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-04T15:15:03,056 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:03,058 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:15:03,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=413 (was 411) - Thread LEAK? -, OpenFileDescriptor=704 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1082) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2459 (was 2544) 2024-12-04T15:15:03,168 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=413, OpenFileDescriptor=704, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2458 2024-12-04T15:15:03,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:03,180 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:03,181 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e895cfe to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47459972 2024-12-04T15:15:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:15:03,242 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x4e895cfe {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4e895cfe to 127.0.0.1:64157 2024-12-04T15:15:03,243 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:03,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:15:03,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132 2024-12-04T15:15:03,441 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:15:03,520 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:03,535 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-04T15:15:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-04T15:15:03,541 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:03,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-04T15:15:03,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-04T15:15:03,555 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:03,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 384 msec 2024-12-04T15:15:03,564 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:03,565 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b42b863 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ad7275d 2024-12-04T15:15:03,592 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:03,592 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-04T15:15:03,805 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:03,808 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:15:03,817 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:03,818 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3e1cd280 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54da3acd 2024-12-04T15:15:03,872 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x3e1cd280 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3e1cd280 to 127.0.0.1:64157 2024-12-04T15:15:03,874 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:15:03,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:04,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-04T15:15:04,102 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:04,107 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:04,107 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:04,108 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:04,108 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:04,110 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x657066c2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b0a74c5 2024-12-04T15:15:04,119 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:15:04,154 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:04,155 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:04,158 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-04T15:15:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-04T15:15:04,162 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:04,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-04T15:15:04,170 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:04,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 215 msec 2024-12-04T15:15:04,235 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:04,237 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dd402f9 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b7b207 2024-12-04T15:15:04,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 362 msec 2024-12-04T15:15:04,270 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:04,271 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-04T15:15:04,436 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:04,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:04,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:04,440 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:04,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:15:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T15:15:04,446 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:04,447 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0682d428 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25fbe73e 2024-12-04T15:15:04,479 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0682d428 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0682d428 to 127.0.0.1:64157 2024-12-04T15:15:04,479 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:04,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T15:15:04,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-04T15:15:04,680 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:04,681 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:04,681 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:04,682 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:04,682 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:04,684 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5abc431e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@281aeb0e 2024-12-04T15:15:04,715 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:04,751 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:04,751 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T15:15:04,774 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-04T15:15:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-04T15:15:04,779 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:04,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-04T15:15:04,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-04T15:15:04,782 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:15:04,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 341 msec 2024-12-04T15:15:04,803 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:04,805 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x717c6a55 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8ec3655 2024-12-04T15:15:04,831 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:04,832 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-04T15:15:05,076 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:05,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,082 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,085 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-04T15:15:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:15:05,101 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:05,102 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fcccbe3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64be3d82 2024-12-04T15:15:05,175 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1fcccbe3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fcccbe3 to 127.0.0.1:64157 2024-12-04T15:15:05,178 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:05,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:15:05,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-04T15:15:05,363 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:05,371 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:05,371 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:05,371 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:05,372 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:05,374 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05336509 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@77d2c9aa 2024-12-04T15:15:05,385 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:15:05,420 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:05,420 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:05,427 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-04T15:15:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-04T15:15:05,435 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:05,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-04T15:15:05,444 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-04T15:15:05,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 231 msec 2024-12-04T15:15:05,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 354 msec 2024-12-04T15:15:05,490 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:05,495 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7c781a80 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2aa5b734 2024-12-04T15:15:05,533 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:05,533 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-04T15:15:05,731 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:05,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:05,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T15:15:05,758 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:05,759 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66e554b2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3df5db78 2024-12-04T15:15:05,778 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x66e554b2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66e554b2 to 127.0.0.1:64157 2024-12-04T15:15:05,778 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:05,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T15:15:05,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-04T15:15:05,955 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:05,964 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:05,964 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:05,964 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:05,965 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:05,966 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4bb3568f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d53580c 2024-12-04T15:15:05,979 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:06,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:06,044 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:06,057 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-04T15:15:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-04T15:15:06,061 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:06,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-04T15:15:06,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec 2024-12-04T15:15:06,064 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:06,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 317 msec 2024-12-04T15:15:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-04T15:15:06,075 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:06,076 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:06,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:06,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-04T15:15:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-04T15:15:06,081 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,083 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03ca2fc4 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27aabc3a 2024-12-04T15:15:06,106 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,107 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x03ca2fc4 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x03ca2fc4 to 127.0.0.1:64157 2024-12-04T15:15:06,107 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:06,120 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65b68835 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@571997c2 2024-12-04T15:15:06,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:06,160 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:06,160 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-04T15:15:06,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-04T15:15:06,292 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:06,293 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:06,293 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:06,293 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:06,293 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,295 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d7285cc to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48025071 2024-12-04T15:15:06,298 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:06,320 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:06,321 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:06,326 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-04T15:15:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-04T15:15:06,339 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:06,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-04T15:15:06,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 202 msec 2024-12-04T15:15:06,341 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-04T15:15:06,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 263 msec 2024-12-04T15:15:06,364 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,365 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6876ffcb to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5dc3425e 2024-12-04T15:15:06,385 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:06,385 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-04T15:15:06,395 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:06,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:06,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-04T15:15:06,401 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,401 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29e7c393 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51a15396 2024-12-04T15:15:06,433 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x29e7c393 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29e7c393 to 127.0.0.1:64157 2024-12-04T15:15:06,438 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:06,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-04T15:15:06,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-04T15:15:06,632 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:06,655 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:06,655 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:06,656 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:06,656 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,660 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ea4eaeb to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f8718af 2024-12-04T15:15:06,667 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-04T15:15:06,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:06,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:06,719 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-04T15:15:06,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-04T15:15:06,730 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:06,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-04T15:15:06,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-04T15:15:06,741 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:06,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 344 msec 2024-12-04T15:15:06,765 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:06,767 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21181312 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21ffcaf7 2024-12-04T15:15:06,807 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:06,807 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:07,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-04T15:15:07,028 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:07,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:07,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:07,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:07,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-04T15:15:07,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:15:07,066 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:07,068 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:07,073 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:07,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:07,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:07,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:07,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:07,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:07,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:07,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-04T15:15:07,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:15:07,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147 2024-12-04T15:15:07,249 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:07,250 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:07,250 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:07,250 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:07,250 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:07,253 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48bb2f77 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d64e74b 2024-12-04T15:15:07,259 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:07,262 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:07,266 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-04T15:15:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-04T15:15:07,276 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:07,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-12-04T15:15:07,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-04T15:15:07,287 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:07,287 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:07,327 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:07,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 293 msec 2024-12-04T15:15:07,339 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:07,343 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=146, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:07,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=146, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=309 msec 2024-12-04T15:15:07,349 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-04T15:15:07,368 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-04T15:15:07,376 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:07,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:15:07,383 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:07,385 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=148, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:07,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=148, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=9 msec 2024-12-04T15:15:07,391 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-04T15:15:07,485 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:07,539 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=454 (was 413) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1d7285cc-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21181312-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7c781a80 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x65b68835-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1d7285cc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x717c6a55 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6876ffcb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x05336509-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21181312-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6876ffcb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x657066c2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd402f9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x717c6a55-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ea4eaeb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48bb2f77-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd402f9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0b42b863-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x657066c2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3dd402f9-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ea4eaeb-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x65b68835-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4bb3568f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4bb3568f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x657066c2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x65b68835 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48bb2f77-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x05336509 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1d7285cc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ea4eaeb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x48bb2f77 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5abc431e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4bb3568f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x05336509-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6876ffcb-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0b42b863-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7c781a80-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5abc431e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x717c6a55-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0b42b863 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5abc431e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x21181312 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7c781a80-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=757 (was 704) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1246 (was 1163) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2379 (was 2458) 2024-12-04T15:15:07,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=454, OpenFileDescriptor=757, MaxFileDescriptor=1048576, SystemLoadAverage=1246, ProcessCount=11, AvailableMemoryMB=2374 2024-12-04T15:15:07,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:07,590 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:07,591 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dd4e9e7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53718d44 2024-12-04T15:15:07,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-04T15:15:07,670 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1dd4e9e7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dd4e9e7 to 127.0.0.1:64157 2024-12-04T15:15:07,670 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:07,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-04T15:15:07,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-04T15:15:07,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=150 2024-12-04T15:15:07,917 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:08,008 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:08,017 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-12-04T15:15:08,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-12-04T15:15:08,021 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:08,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-04T15:15:08,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-04T15:15:08,031 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:08,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 450 msec 2024-12-04T15:15:08,052 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:08,054 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ab36059 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f11f785 2024-12-04T15:15:08,141 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:08,141 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-04T15:15:08,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:08,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-04T15:15:08,234 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:08,236 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f2e8107 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46348730 2024-12-04T15:15:08,294 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x7f2e8107 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7f2e8107 to 127.0.0.1:64157 2024-12-04T15:15:08,294 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-04T15:15:08,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:08,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-04T15:15:08,500 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:08,506 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:08,506 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:08,506 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:08,506 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:08,508 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x220c9081 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@197c9efa 2024-12-04T15:15:08,509 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-04T15:15:08,562 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:08,562 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:08,583 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-04T15:15:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-12-04T15:15:08,591 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:08,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-04T15:15:08,594 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:08,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 254 msec 2024-12-04T15:15:08,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 375 msec 2024-12-04T15:15:08,608 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:08,610 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6680b36f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a98d392 2024-12-04T15:15:08,645 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:08,646 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-04T15:15:08,862 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:08,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:15:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:08,869 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-04T15:15:08,873 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1871d3cb to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34c71dde 2024-12-04T15:15:08,911 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1871d3cb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1871d3cb to 127.0.0.1:64157 2024-12-04T15:15:08,911 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-04T15:15:08,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:09,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-04T15:15:09,145 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:09,154 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:09,154 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:09,154 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:09,155 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,156 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1bd04ed1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@415e9bb2 2024-12-04T15:15:09,163 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-04T15:15:09,206 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:09,207 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:09,210 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-04T15:15:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=154 2024-12-04T15:15:09,213 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:09,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-04T15:15:09,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 225 msec 2024-12-04T15:15:09,216 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:15:09,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 353 msec 2024-12-04T15:15:09,232 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,234 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35b0b14c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b6a4c94 2024-12-04T15:15:09,254 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:09,255 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-04T15:15:09,495 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:09,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-04T15:15:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-04T15:15:09,508 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,509 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1821e36c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f371b2a 2024-12-04T15:15:09,547 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1821e36c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1821e36c to 127.0.0.1:64157 2024-12-04T15:15:09,547 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:09,547 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:09,548 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=155, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:09,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=155, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=53 msec 2024-12-04T15:15:09,550 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-04T15:15:09,605 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-04T15:15:09,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:15:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:15:09,612 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,613 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61a86720 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45e31e09 2024-12-04T15:15:09,647 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x61a86720 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61a86720 to 127.0.0.1:64157 2024-12-04T15:15:09,647 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:09,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:15:09,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=157 2024-12-04T15:15:09,821 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:09,838 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:09,838 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:09,838 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:09,838 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,840 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5eaf62d7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3240fbc3 2024-12-04T15:15:09,843 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:09,876 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:09,876 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:09,886 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-04T15:15:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-04T15:15:09,890 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:09,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-04T15:15:09,892 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:15:09,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 224 msec 2024-12-04T15:15:09,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 286 msec 2024-12-04T15:15:09,914 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,915 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1643ee4d to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42c25043 2024-12-04T15:15:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-04T15:15:09,925 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:09,927 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-04T15:15:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:15:09,935 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:09,936 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59d9f140 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5972dbf5 2024-12-04T15:15:09,946 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:09,946 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:09,963 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x59d9f140 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x59d9f140 to 127.0.0.1:64157 2024-12-04T15:15:09,963 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:09,963 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:09,965 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=158, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:09,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=158, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=38 msec 2024-12-04T15:15:09,966 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-04T15:15:10,036 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-04T15:15:10,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-04T15:15:10,068 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:10,075 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:10,077 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:10,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:10,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:10,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:10,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:10,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:10,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-04T15:15:10,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=160 2024-12-04T15:15:10,311 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:10,311 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:10,311 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:10,313 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:10,313 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:10,315 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30b11b03 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3cb34037 2024-12-04T15:15:10,317 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:10,318 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:10,328 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160 2024-12-04T15:15:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=160 2024-12-04T15:15:10,333 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:10,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-04T15:15:10,337 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:10,337 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:10,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-04T15:15:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-04T15:15:10,375 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:10,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 338 msec 2024-12-04T15:15:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-04T15:15:10,685 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:10,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:10,690 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-04T15:15:10,692 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=161, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:10,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=161, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:15:10,693 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-04T15:15:10,795 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:10,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:15:10,807 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:10,809 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=162, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:10,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=162, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-04T15:15:10,811 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-04T15:15:10,916 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:10,984 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=478 (was 454) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1643ee4d-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6680b36f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1bd04ed1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1bd04ed1-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x35b0b14c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1643ee4d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1ab36059-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1643ee4d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1ab36059 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x220c9081-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6680b36f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1bd04ed1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5eaf62d7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5eaf62d7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x30b11b03 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6680b36f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5eaf62d7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x220c9081-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x35b0b14c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1ab36059-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x30b11b03-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x220c9081 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x35b0b14c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x30b11b03-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=789 (was 757) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1259 (was 1246) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2346 (was 2374) 2024-12-04T15:15:11,012 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=478, OpenFileDescriptor=789, MaxFileDescriptor=1048576, SystemLoadAverage=1259, ProcessCount=11, AvailableMemoryMB=2344 2024-12-04T15:15:11,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:11,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-04T15:15:11,024 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:11,025 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x326a2859 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53fa4a95 2024-12-04T15:15:11,127 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x326a2859 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x326a2859 to 127.0.0.1:64157 2024-12-04T15:15:11,130 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-04T15:15:11,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:11,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=164 2024-12-04T15:15:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-04T15:15:11,357 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:11,441 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:11,451 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164 2024-12-04T15:15:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=164 2024-12-04T15:15:11,457 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:11,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-04T15:15:11,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-12-04T15:15:11,461 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:11,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 445 msec 2024-12-04T15:15:11,495 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:11,502 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x733f234f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a93ce7f 2024-12-04T15:15:11,534 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:11,535 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-04T15:15:11,664 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:11,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:11,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:11,670 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:11,671 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38fa4a1f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@215eaf7d 2024-12-04T15:15:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-04T15:15:11,729 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x38fa4a1f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38fa4a1f to 127.0.0.1:64157 2024-12-04T15:15:11,730 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:11,730 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:11,737 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=165, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:11,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=165, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=72 msec 2024-12-04T15:15:11,741 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-04T15:15:11,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-04T15:15:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:11,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:15:11,788 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:11,794 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=166, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:11,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=166, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=13 msec 2024-12-04T15:15:11,799 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-04T15:15:11,896 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:11,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:11,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-04T15:15:11,908 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:15:11,909 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5554d4c8 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@77a5edea 2024-12-04T15:15:11,943 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x5554d4c8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5554d4c8 to 127.0.0.1:64157 2024-12-04T15:15:11,945 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:11,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-04T15:15:12,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=168 2024-12-04T15:15:12,134 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-04T15:15:12,207 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.ReplicationSource(231): queueId=2-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 2, currentBandwidth=0 2024-12-04T15:15:12,212 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=168 2024-12-04T15:15:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=168 2024-12-04T15:15:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-04T15:15:12,224 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:12,231 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:12,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-04T15:15:12,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 250 msec 2024-12-04T15:15:12,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 333 msec 2024-12-04T15:15:12,236 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:15:12,239 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c31117f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@173b1273 2024-12-04T15:15:12,302 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:12,303 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-04T15:15:12,531 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-04T15:15:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:12,535 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:12,537 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x113b0655 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a438aae 2024-12-04T15:15:12,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-04T15:15:12,551 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:12,559 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:12,565 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:12,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:12,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:12,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:12,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:12,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:12,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:12,583 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:12,584 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-04T15:15:12,643 INFO [master/c6f2e1d35cf1:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T15:15:12,643 INFO [master/c6f2e1d35cf1:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T15:15:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-04T15:15:12,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-04T15:15:12,738 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:12,738 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:12,738 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:12,739 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:12,739 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:12,741 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2462265c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d7f246e 2024-12-04T15:15:12,752 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:12,753 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:12,758 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-04T15:15:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=170 2024-12-04T15:15:12,762 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:12,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:15:12,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-04T15:15:12,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-04T15:15:12,765 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:12,765 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:12,778 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:12,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 238 msec 2024-12-04T15:15:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-04T15:15:12,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:12,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-04T15:15:12,884 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-04T15:15:12,888 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-04T15:15:12,895 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-04T15:15:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-04T15:15:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-04T15:15:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-04T15:15:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-04T15:15:12,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-04T15:15:12,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-04T15:15:13,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-04T15:15:13,067 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-04T15:15:13,067 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-04T15:15:13,067 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:13,067 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-04T15:15:13,067 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test2' 2024-12-04T15:15:13,069 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23272dc3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37f42515 2024-12-04T15:15:13,073 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,2-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:13,074 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(562): Done with the queue 2-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:13,081 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-04T15:15:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=172 2024-12-04T15:15:13,086 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:13,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-04T15:15:13,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:15:13,090 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-04T15:15:13,090 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-04T15:15:13,105 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-04T15:15:13,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 229 msec 2024-12-04T15:15:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-04T15:15:13,197 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-04T15:15:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:13,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:13,207 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:13,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-04T15:15:13,209 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=173, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=173, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-04T15:15:13,214 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-04T15:15:13,323 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:13,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-04T15:15:13,335 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:13,336 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=174, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=174, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-04T15:15:13,338 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-04T15:15:13,436 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:13,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=491 (was 478) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x113b0655-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0c31117f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x733f234f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x23272dc3-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x113b0655-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2462265c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0c31117f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x113b0655 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x23272dc3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x733f234f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x733f234f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0c31117f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2462265c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2462265c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x23272dc3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=805 (was 789) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1259 (was 1259), ProcessCount=11 (was 11), AvailableMemoryMB=2305 (was 2344) 2024-12-04T15:15:13,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=491, OpenFileDescriptor=805, MaxFileDescriptor=1048576, SystemLoadAverage=1259, ProcessCount=11, AvailableMemoryMB=2304 2024-12-04T15:15:13,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:13,530 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-04T15:15:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-04T15:15:13,540 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=175, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=175, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=15 msec 2024-12-04T15:15:13,544 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-04T15:15:13,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-04T15:15:13,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:13,657 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:13,660 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=176, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=176, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-04T15:15:13,662 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-04T15:15:13,667 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:13,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:13,675 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-04T15:15:13,677 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=177, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=177, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-04T15:15:13,679 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-04T15:15:13,786 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:13,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=491 (was 491), OpenFileDescriptor=805 (was 805), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1259 (was 1259), ProcessCount=11 (was 11), AvailableMemoryMB=2303 (was 2304) 2024-12-04T15:15:13,843 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=491, OpenFileDescriptor=805, MaxFileDescriptor=1048576, SystemLoadAverage=1259, ProcessCount=11, AvailableMemoryMB=2302 2024-12-04T15:15:13,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-04T15:15:13,855 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-04T15:15:13,858 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=178, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:13,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=178, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=14 msec 2024-12-04T15:15:13,860 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-04T15:15:13,956 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-04T15:15:13,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-04T15:15:14,002 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:14,006 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=179, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:14,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=179, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=47 msec 2024-12-04T15:15:14,009 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-04T15:15:14,106 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:14,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-04T15:15:14,110 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:14,112 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=180, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:14,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=180, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:15:14,114 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-04T15:15:14,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:14,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=491 (was 491), OpenFileDescriptor=805 (was 805), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1259 (was 1259), ProcessCount=11 (was 11), AvailableMemoryMB=2308 (was 2302) - AvailableMemoryMB LEAK? - 2024-12-04T15:15:14,273 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=491, OpenFileDescriptor=807, MaxFileDescriptor=1048576, SystemLoadAverage=1259, ProcessCount=11, AvailableMemoryMB=2307 2024-12-04T15:15:14,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-04T15:15:14,280 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:14,281 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0241b322 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@16c7b113 2024-12-04T15:15:14,304 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0241b322 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0241b322 to 127.0.0.1:64157 2024-12-04T15:15:14,305 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:14,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-04T15:15:14,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=182 2024-12-04T15:15:14,538 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-04T15:15:14,617 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:14,635 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=182 2024-12-04T15:15:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=182 2024-12-04T15:15:14,651 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:14,671 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:14,673 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02f8acd0 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14063cc 2024-12-04T15:15:14,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-12-04T15:15:14,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-04T15:15:14,687 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:14,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 412 msec 2024-12-04T15:15:14,746 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:14,747 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-04T15:15:14,905 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:14,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-04T15:15:14,909 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:14,911 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c23c40f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a91ed8a 2024-12-04T15:15:14,927 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0c23c40f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c23c40f to 127.0.0.1:64157 2024-12-04T15:15:14,928 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:14,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-04T15:15:15,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-04T15:15:15,106 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:15,107 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:15,107 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:15,110 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:15,111 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,113 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24074132 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a00018a 2024-12-04T15:15:15,113 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:15,149 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:15,150 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:15,154 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-04T15:15:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=184 2024-12-04T15:15:15,167 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:15,180 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-12-04T15:15:15,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 214 msec 2024-12-04T15:15:15,181 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:15,182 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5b4d1555 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15cf100 2024-12-04T15:15:15,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 275 msec 2024-12-04T15:15:15,215 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:15,215 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-04T15:15:15,228 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:15,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-04T15:15:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:15,238 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,240 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60d86a7a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@570053f1 2024-12-04T15:15:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-04T15:15:15,290 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x60d86a7a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60d86a7a to 127.0.0.1:64157 2024-12-04T15:15:15,291 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:15,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-04T15:15:15,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-04T15:15:15,475 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:15,476 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:15,476 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:15,476 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:15,476 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,478 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x289b1bf1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5655daac 2024-12-04T15:15:15,478 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:15,504 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:15,505 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:15,506 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-04T15:15:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=186 2024-12-04T15:15:15,524 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:15,537 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=186, resume processing ppid=185 2024-12-04T15:15:15,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, ppid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-12-04T15:15:15,538 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-04T15:15:15,539 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55eff098 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25a19f4a 2024-12-04T15:15:15,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 307 msec 2024-12-04T15:15:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-04T15:15:15,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:15,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:15,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:15:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-04T15:15:15,574 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,575 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f5ca310 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7bf247b0 2024-12-04T15:15:15,577 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:15,577 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:15,603 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x2f5ca310 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f5ca310 to 127.0.0.1:64157 2024-12-04T15:15:15,603 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:15,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-04T15:15:15,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-04T15:15:15,809 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:15,810 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:15,810 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:15,810 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:15,811 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,813 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c6888f7 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@378884cf 2024-12-04T15:15:15,819 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:15,841 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:15,841 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:15,841 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-04T15:15:15,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=188 2024-12-04T15:15:15,855 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:15,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=188, resume processing ppid=187 2024-12-04T15:15:15,857 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-04T15:15:15,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, ppid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 208 msec 2024-12-04T15:15:15,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 293 msec 2024-12-04T15:15:15,859 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:15,860 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b7b3362 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4354a787 2024-12-04T15:15:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-04T15:15:15,890 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:15,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:15,896 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:15,897 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:15,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:15,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-04T15:15:15,929 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:15,934 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:15,943 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:15,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:15,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:15,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:15,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:15,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:15,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-04T15:15:16,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-04T15:15:16,146 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:16,146 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:16,146 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:16,146 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:16,146 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:16,148 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4338cfe9 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a70827d 2024-12-04T15:15:16,148 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:16,148 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:16,152 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-04T15:15:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=190 2024-12-04T15:15:16,169 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:16,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=190, resume processing ppid=189 2024-12-04T15:15:16,186 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:16,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, ppid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:15:16,186 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:16,206 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:16,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 304 msec 2024-12-04T15:15:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-04T15:15:16,233 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:16,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-04T15:15:16,236 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:16,239 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=191, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:16,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=191, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:15:16,241 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-04T15:15:16,345 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:16,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=192, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-04T15:15:16,349 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:16,350 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=192, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:16,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=192, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:15:16,352 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-04T15:15:16,455 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:16,476 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=515 (was 491) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55eff098-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55eff098 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b7b3362 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x02f8acd0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5b4d1555-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x289b1bf1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4338cfe9-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c6888f7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x289b1bf1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x24074132 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b7b3362-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x02f8acd0-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6b7b3362-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c6888f7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5b4d1555-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c6888f7-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x24074132-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x02f8acd0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4338cfe9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4338cfe9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5b4d1555 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x289b1bf1-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55eff098-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x24074132-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=837 (was 807) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1238 (was 1259), ProcessCount=11 (was 11), AvailableMemoryMB=2280 (was 2307) 2024-12-04T15:15:16,479 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=515 is superior to 500 2024-12-04T15:15:16,496 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=515, OpenFileDescriptor=837, MaxFileDescriptor=1048576, SystemLoadAverage=1238, ProcessCount=11, AvailableMemoryMB=2279 2024-12-04T15:15:16,496 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=515 is superior to 500 2024-12-04T15:15:16,497 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-04T15:15:16,501 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:16,503 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x205f17f2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55a7484b 2024-12-04T15:15:16,515 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x205f17f2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x205f17f2 to 127.0.0.1:64157 2024-12-04T15:15:16,515 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:16,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=194, ppid=193, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-04T15:15:16,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=194 2024-12-04T15:15:16,698 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:16,738 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:16,740 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=194 2024-12-04T15:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=194 2024-12-04T15:15:16,742 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:16,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=194, resume processing ppid=193 2024-12-04T15:15:16,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=194, ppid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-04T15:15:16,744 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:16,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 246 msec 2024-12-04T15:15:16,757 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:16,760 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x267539a0 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15a6b192 2024-12-04T15:15:16,770 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:16,771 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-04T15:15:16,814 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:16,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-04T15:15:16,818 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:16,818 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cf1b740 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2dab1148 2024-12-04T15:15:16,827 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x1cf1b740 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1cf1b740 to 127.0.0.1:64157 2024-12-04T15:15:16,827 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:16,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-04T15:15:16,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-04T15:15:16,997 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:16,999 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:16,999 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:16,999 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:16,999 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,001 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0eedd712 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5193a958 2024-12-04T15:15:17,002 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:17,029 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:17,029 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:17,030 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-04T15:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=196 2024-12-04T15:15:17,033 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:17,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=196, resume processing ppid=195 2024-12-04T15:15:17,044 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-04T15:15:17,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=196, ppid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-04T15:15:17,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 229 msec 2024-12-04T15:15:17,050 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,051 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58464d38 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@415f6374 2024-12-04T15:15:17,060 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:17,061 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-04T15:15:17,135 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:17,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:17,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:15:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-04T15:15:17,141 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,142 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3212d655 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c4347c1 2024-12-04T15:15:17,158 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x3212d655 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3212d655 to 127.0.0.1:64157 2024-12-04T15:15:17,159 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:17,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-04T15:15:17,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-04T15:15:17,330 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:17,332 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:17,332 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:17,332 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:17,332 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,333 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70b76ae1 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e08b979 2024-12-04T15:15:17,334 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:17,353 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:17,353 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:17,353 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-04T15:15:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=198 2024-12-04T15:15:17,356 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:17,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=198, resume processing ppid=197 2024-12-04T15:15:17,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=198, ppid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-04T15:15:17,359 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-04T15:15:17,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 222 msec 2024-12-04T15:15:17,374 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,375 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5d0383ff to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@382a52e6 2024-12-04T15:15:17,395 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:17,395 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-04T15:15:17,454 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:17,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:17,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:17,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:15:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-04T15:15:17,461 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,462 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d62db19 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b11af1c 2024-12-04T15:15:17,473 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x2d62db19 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d62db19 to 127.0.0.1:64157 2024-12-04T15:15:17,473 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:17,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-04T15:15:17,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-04T15:15:17,644 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:17,645 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:17,645 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:17,645 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:17,646 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,647 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f307c03 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76a464a1 2024-12-04T15:15:17,648 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:17,669 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:17,669 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:17,670 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-04T15:15:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=200 2024-12-04T15:15:17,673 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:17,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=200, resume processing ppid=199 2024-12-04T15:15:17,675 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:15:17,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=200, ppid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-04T15:15:17,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec 2024-12-04T15:15:17,690 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,692 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x314bcae6 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e7a1f6f 2024-12-04T15:15:17,723 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:17,723 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-04T15:15:17,775 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:17,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:17,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:17,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:15:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-04T15:15:17,781 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,782 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57db95b3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@770b03ae 2024-12-04T15:15:17,798 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x57db95b3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57db95b3 to 127.0.0.1:64157 2024-12-04T15:15:17,798 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:17,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-04T15:15:17,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-04T15:15:17,969 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:17,970 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:17,971 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:17,971 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:17,971 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:17,973 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26589a33 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@721d940b 2024-12-04T15:15:17,973 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:17,997 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:17,997 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:17,997 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-04T15:15:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=202 2024-12-04T15:15:18,001 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:18,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=202, resume processing ppid=201 2024-12-04T15:15:18,004 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-04T15:15:18,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=202, ppid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-04T15:15:18,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 226 msec 2024-12-04T15:15:18,019 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,021 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d32ade9 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@19333421 2024-12-04T15:15:18,032 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:18,032 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-04T15:15:18,094 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:18,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:18,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:18,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-04T15:15:18,100 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,102 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d566996 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64598433 2024-12-04T15:15:18,110 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x7d566996 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d566996 to 127.0.0.1:64157 2024-12-04T15:15:18,111 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:18,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-04T15:15:18,255 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:15:18,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-04T15:15:18,281 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:18,282 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:18,282 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:18,282 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:18,282 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,283 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4dd5521c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23a0a109 2024-12-04T15:15:18,284 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:18,300 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:18,300 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:18,301 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-04T15:15:18,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=204 2024-12-04T15:15:18,304 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:18,306 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=204, resume processing ppid=203 2024-12-04T15:15:18,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=204, ppid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:15:18,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-04T15:15:18,317 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,319 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00033c3e to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d780945 2024-12-04T15:15:18,327 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:18,327 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-04T15:15:18,414 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:18,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:18,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:18,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-04T15:15:18,420 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,422 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4abd3517 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4abafc1e 2024-12-04T15:15:18,431 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x4abd3517 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4abd3517 to 127.0.0.1:64157 2024-12-04T15:15:18,432 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:18,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-04T15:15:18,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-04T15:15:18,601 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:18,602 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:18,602 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:18,602 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:18,603 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,605 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0121026a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64fc514 2024-12-04T15:15:18,605 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:18,628 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:18,628 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:18,628 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-04T15:15:18,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=206 2024-12-04T15:15:18,631 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:18,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=206, resume processing ppid=205 2024-12-04T15:15:18,633 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=206, ppid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-04T15:15:18,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 216 msec 2024-12-04T15:15:18,649 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,651 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x170359a2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72c0341c 2024-12-04T15:15:18,698 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:18,698 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-04T15:15:18,735 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:18,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:18,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-04T15:15:18,740 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:18,741 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x688ce70c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63e6dd6f 2024-12-04T15:15:18,752 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x688ce70c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x688ce70c to 127.0.0.1:64157 2024-12-04T15:15:18,752 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:18,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-04T15:15:18,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-04T15:15:18,922 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:18,924 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-04T15:15:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=208 2024-12-04T15:15:18,927 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:18,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=208, resume processing ppid=207 2024-12-04T15:15:18,929 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-04T15:15:18,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=208, ppid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 158 msec 2024-12-04T15:15:18,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 192 msec 2024-12-04T15:15:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-04T15:15:19,055 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:19,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:19,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:19,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-04T15:15:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-04T15:15:19,059 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,060 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x44df03cd to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c63d05 2024-12-04T15:15:19,069 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x44df03cd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x44df03cd to 127.0.0.1:64157 2024-12-04T15:15:19,069 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:19,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-04T15:15:19,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-04T15:15:19,241 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:19,242 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:19,242 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:19,242 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:19,242 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,243 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42013c8a to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29f42693 2024-12-04T15:15:19,244 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:19,260 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:19,260 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:19,261 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-04T15:15:19,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=210 2024-12-04T15:15:19,263 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:19,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=210, resume processing ppid=209 2024-12-04T15:15:19,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=210, ppid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-04T15:15:19,266 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-04T15:15:19,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 209 msec 2024-12-04T15:15:19,275 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,276 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07ea66f6 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69a0e26c 2024-12-04T15:15:19,311 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:19,311 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-04T15:15:19,375 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:19,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:19,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-04T15:15:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-04T15:15:19,379 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,380 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x687592a3 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@441dd396 2024-12-04T15:15:19,398 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x687592a3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x687592a3 to 127.0.0.1:64157 2024-12-04T15:15:19,398 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:19,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-04T15:15:19,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-04T15:15:19,624 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:19,625 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:19,625 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:19,625 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:19,625 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,626 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x12786c55 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71b67370 2024-12-04T15:15:19,626 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:19,644 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:19,645 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:19,647 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-04T15:15:19,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=212 2024-12-04T15:15:19,653 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:19,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=212, resume processing ppid=211 2024-12-04T15:15:19,661 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-04T15:15:19,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=212, ppid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-04T15:15:19,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 285 msec 2024-12-04T15:15:19,667 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,668 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2586c1df to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d17de6 2024-12-04T15:15:19,681 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:19,681 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-04T15:15:19,695 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:19,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:19,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:19,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-04T15:15:19,700 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:19,700 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:19,701 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:19,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:19,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:19,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:19,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:19,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:19,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-04T15:15:19,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-04T15:15:19,863 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:19,863 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:19,863 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:19,863 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:19,863 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:19,864 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fad295b to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fa2fc71 2024-12-04T15:15:19,864 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:19,865 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:19,867 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-04T15:15:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=214 2024-12-04T15:15:19,868 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:19,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=214, resume processing ppid=213 2024-12-04T15:15:19,869 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:19,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=214, ppid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-12-04T15:15:19,869 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:19,876 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:19,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 180 msec 2024-12-04T15:15:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-04T15:15:20,014 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:20,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:20,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-04T15:15:20,017 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:20,017 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=215, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:20,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=215, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-04T15:15:20,018 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-04T15:15:20,125 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:20,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=216, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:20,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-04T15:15:20,128 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:20,129 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=216, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:20,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=216, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:15:20,130 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-04T15:15:20,235 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:20,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=569 (was 515) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0eedd712 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x314bcae6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d32ade9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x170359a2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1fad295b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x314bcae6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x314bcae6-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x07ea66f6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70b76ae1-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4dd5521c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x07ea66f6-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x26589a33-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4dd5521c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x26589a33-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x00033c3e-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d32ade9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x12786c55-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x00033c3e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x170359a2-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x07ea66f6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5d0383ff-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70b76ae1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1fad295b-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x12786c55-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x42013c8a-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0eedd712-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x42013c8a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x170359a2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5d0383ff-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0eedd712-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x70b76ae1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x12786c55 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x58464d38-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x267539a0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x267539a0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0121026a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x5d0383ff java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x00033c3e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x267539a0-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6f307c03 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4dd5521c-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0121026a-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x58464d38-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2586c1df-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2586c1df java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6f307c03-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d32ade9-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x2586c1df-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x6f307c03-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x1fad295b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0121026a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x58464d38 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x26589a33 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x42013c8a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=909 (was 837) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1238), ProcessCount=11 (was 11), AvailableMemoryMB=2279 (was 2279) 2024-12-04T15:15:20,255 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=569 is superior to 500 2024-12-04T15:15:20,271 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=569, OpenFileDescriptor=909, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2278 2024-12-04T15:15:20,271 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=569 is superior to 500 2024-12-04T15:15:20,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-04T15:15:20,275 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:20,276 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7daa33d2 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ac429f 2024-12-04T15:15:20,318 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x7daa33d2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7daa33d2 to 127.0.0.1:64157 2024-12-04T15:15:20,319 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-04T15:15:20,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=218, ppid=217, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-04T15:15:20,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=218 2024-12-04T15:15:20,629 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:20,667 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:20,669 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=218 2024-12-04T15:15:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=218 2024-12-04T15:15:20,671 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:20,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=218, resume processing ppid=217 2024-12-04T15:15:20,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=218, ppid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-04T15:15:20,673 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:20,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 401 msec 2024-12-04T15:15:20,682 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:20,683 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29782fcb to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c1e4401 2024-12-04T15:15:20,736 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:20,737 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-04T15:15:20,905 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:20,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-04T15:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-04T15:15:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-04T15:15:20,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-04T15:15:21,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-04T15:15:21,080 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-04T15:15:21,081 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-04T15:15:21,081 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-04T15:15:21,081 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:21,081 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:21,082 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x200cdbca to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71d3ef3 2024-12-04T15:15:21,082 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:21,097 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:21,097 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-04T15:15:21,097 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-04T15:15:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=220 2024-12-04T15:15:21,099 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:21,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=220, resume processing ppid=219 2024-12-04T15:15:21,101 INFO [PEWorker-3 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-04T15:15:21,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=220, ppid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 172 msec 2024-12-04T15:15:21,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 194 msec 2024-12-04T15:15:21,111 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:21,112 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x44080d67 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@158fe9f2 2024-12-04T15:15:21,184 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:21,184 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-04T15:15:21,225 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:21,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-04T15:15:21,234 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:21,234 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:21,235 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-04T15:15:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:21,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:21,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-04T15:15:21,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-04T15:15:21,573 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:21,574 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:21,574 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:21,574 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:21,574 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:21,575 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ecaaa85 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@66645c65 2024-12-04T15:15:21,575 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:21,576 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:21,577 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-04T15:15:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=222 2024-12-04T15:15:21,579 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:21,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=222, resume processing ppid=221 2024-12-04T15:15:21,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=222, ppid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-04T15:15:21,581 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:21,581 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:21,586 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:21,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 359 msec 2024-12-04T15:15:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-04T15:15:21,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:21,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-04T15:15:21,860 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:21,862 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=223, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:21,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=223, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-04T15:15:21,864 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-04T15:15:21,965 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:21,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=224, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-04T15:15:21,968 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:21,969 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=224, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:21,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=224, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:15:21,970 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-04T15:15:22,075 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:22,092 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=581 (was 569) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ecaaa85-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x200cdbca java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x29782fcb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x44080d67-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ecaaa85 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x44080d67 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x29782fcb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x44080d67-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x200cdbca-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3ecaaa85-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x200cdbca-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x29782fcb-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=925 (was 909) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1163), ProcessCount=11 (was 11), AvailableMemoryMB=2262 (was 2278) 2024-12-04T15:15:22,093 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500 2024-12-04T15:15:22,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=581, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2261 2024-12-04T15:15:22,105 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=581 is superior to 500 2024-12-04T15:15:22,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-04T15:15:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-04T15:15:22,107 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:22,108 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=225, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:22,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=225, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-04T15:15:22,109 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-04T15:15:22,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-04T15:15:22,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-04T15:15:22,221 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:22,223 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=226, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:22,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=226, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-04T15:15:22,225 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-04T15:15:22,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:22,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-04T15:15:22,327 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:22,328 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=227, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:22,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=227, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-04T15:15:22,328 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-04T15:15:22,435 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:22,451 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=579 (was 581), OpenFileDescriptor=925 (was 925), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1163), ProcessCount=11 (was 11), AvailableMemoryMB=2257 (was 2261) 2024-12-04T15:15:22,451 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-04T15:15:22,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=579, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2256 2024-12-04T15:15:22,463 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-04T15:15:22,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-04T15:15:22,467 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:22,468 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30100163 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@335d8070 2024-12-04T15:15:22,502 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x30100163 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x30100163 to 127.0.0.1:64157 2024-12-04T15:15:22,502 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:22,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=229, ppid=228, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-04T15:15:22,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=229 2024-12-04T15:15:22,686 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:22,727 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:22,730 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=229 2024-12-04T15:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=229 2024-12-04T15:15:22,732 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:22,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=229, resume processing ppid=228 2024-12-04T15:15:22,734 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:22,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=229, ppid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec 2024-12-04T15:15:22,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 270 msec 2024-12-04T15:15:22,744 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:22,745 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40e950df to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67ef6bda 2024-12-04T15:15:22,756 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:22,756 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-04T15:15:22,784 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-04T15:15:22,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-04T15:15:22,789 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:22,789 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:22,790 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:22,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:22,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:22,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:22,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:22,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:22,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-04T15:15:22,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-04T15:15:22,977 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:22,977 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:22,977 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:22,977 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:22,978 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:22,979 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d8f1660 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c53306 2024-12-04T15:15:22,980 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:22,980 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:22,983 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-04T15:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=231 2024-12-04T15:15:22,987 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:22,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=231, resume processing ppid=230 2024-12-04T15:15:22,989 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:22,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=231, ppid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-04T15:15:22,989 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:22,996 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:22,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 210 msec 2024-12-04T15:15:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-04T15:15:23,105 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:23,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-04T15:15:23,111 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,113 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=232, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:23,115 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=232, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-04T15:15:23,115 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-04T15:15:23,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:23,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=233, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-04T15:15:23,220 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,222 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=233, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:23,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=233, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:15:23,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-04T15:15:23,325 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:23,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=584 (was 579) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x40e950df java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x40e950df-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d8f1660-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d8f1660 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x7d8f1660-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x40e950df-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=933 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1163), ProcessCount=11 (was 11), AvailableMemoryMB=2238 (was 2256) 2024-12-04T15:15:23,343 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2024-12-04T15:15:23,356 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=582, OpenFileDescriptor=933, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2237 2024-12-04T15:15:23,356 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-04T15:15:23,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 709 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:52758 deadline: 1733325383359, exception=java.io.IOException: Replication peer modification disabled 2024-12-04T15:15:23,360 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T15:15:23,450 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T15:15:23,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] ipc.CallRunner(138): callId: 710 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:52758 deadline: 1733325383464, exception=java.io.IOException: Replication peer modification disabled 2024-12-04T15:15:23,465 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 106 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-04T15:15:23,465 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-04T15:15:23.360Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:15:23.465Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-04T15:15:23,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:23,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-04T15:15:23,468 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,469 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=234, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:23,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=234, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-04T15:15:23,469 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-04T15:15:23,575 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:23,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:23,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-04T15:15:23,577 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:23,578 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=235, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:23,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=235, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-04T15:15:23,579 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-04T15:15:23,685 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:23,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=582 (was 582), OpenFileDescriptor=933 (was 933), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1163), ProcessCount=11 (was 11), AvailableMemoryMB=2223 (was 2237) 2024-12-04T15:15:23,704 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-04T15:15:23,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=582, OpenFileDescriptor=935, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2221 2024-12-04T15:15:23,717 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-04T15:15:23,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-04T15:15:23,725 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:23,726 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ef60430 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15d70546 2024-12-04T15:15:23,747 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x7ef60430 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ef60430 to 127.0.0.1:64157 2024-12-04T15:15:23,747 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:23,777 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=237, ppid=236, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-04T15:15:23,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=237 2024-12-04T15:15:23,930 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:23,960 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:23,963 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=237 2024-12-04T15:15:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=237 2024-12-04T15:15:23,965 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:23,968 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:23,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=237, resume processing ppid=236 2024-12-04T15:15:23,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=237, ppid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-04T15:15:23,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 250 msec 2024-12-04T15:15:23,979 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:23,980 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55f661ca to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e9054fd 2024-12-04T15:15:23,993 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:23,994 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-04T15:15:24,045 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:24,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:24,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:24,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-04T15:15:24,050 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:24,050 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:24,051 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:24,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-04T15:15:24,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-04T15:15:24,214 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:24,214 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:24,214 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:24,215 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:24,215 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:24,216 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4602631d to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4eb7018d 2024-12-04T15:15:24,217 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:24,217 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:24,227 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-04T15:15:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=239 2024-12-04T15:15:24,230 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:24,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=239, resume processing ppid=238 2024-12-04T15:15:24,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=239, ppid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 169 msec 2024-12-04T15:15:24,241 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:24,241 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:24,273 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:24,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 227 msec 2024-12-04T15:15:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-04T15:15:24,364 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:24,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-04T15:15:24,368 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:24,370 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=240, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:24,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=240, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:15:24,371 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-04T15:15:24,475 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:24,497 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=590 (was 582) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4602631d-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55f661ca-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4602631d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55f661ca java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x55f661ca-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x4602631d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=943 (was 935) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1163 (was 1163), ProcessCount=11 (was 11), AvailableMemoryMB=2210 (was 2221) 2024-12-04T15:15:24,497 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=590 is superior to 500 2024-12-04T15:15:24,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=590, OpenFileDescriptor=943, MaxFileDescriptor=1048576, SystemLoadAverage=1163, ProcessCount=11, AvailableMemoryMB=2209 2024-12-04T15:15:24,514 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=590 is superior to 500 2024-12-04T15:15:24,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=241, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-04T15:15:24,518 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:24,519 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e87360c to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@377ded2e 2024-12-04T15:15:24,539 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x0e87360c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e87360c to 127.0.0.1:64157 2024-12-04T15:15:24,539 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-04T15:15:24,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=242, ppid=241, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:24,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=242 2024-12-04T15:15:24,790 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-04T15:15:24,859 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:24,871 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=242 2024-12-04T15:15:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=242 2024-12-04T15:15:24,874 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:24,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=242, resume processing ppid=241 2024-12-04T15:15:24,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=242, ppid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 248 msec 2024-12-04T15:15:24,878 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:24,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 363 msec 2024-12-04T15:15:24,897 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:24,898 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c16bf3f to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32af365c 2024-12-04T15:15:24,931 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-04T15:15:24,932 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-04T15:15:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-04T15:15:25,151 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:25,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:25,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-04T15:15:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-04T15:15:25,162 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:25,163 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x28031014 to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b472964 2024-12-04T15:15:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-04T15:15:25,191 DEBUG [ReadOnlyZKClient-127.0.0.1:64157@0x28031014 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x28031014 to 127.0.0.1:64157 2024-12-04T15:15:25,193 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T15:15:25,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-04T15:15:25,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-04T15:15:25,367 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:15:25,367 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-04T15:15:25,369 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-04T15:15:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=244 2024-12-04T15:15:25,377 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:25,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=244, resume processing ppid=243 2024-12-04T15:15:25,382 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64157/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-04T15:15:25,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=244, ppid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-04T15:15:25,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 223 msec 2024-12-04T15:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-04T15:15:25,485 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-04T15:15:25,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-04T15:15:25,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-04T15:15:25,495 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:25,495 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:25,495 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:25,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:25,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-04T15:15:25,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-04T15:15:25,746 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:25,746 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:25,746 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:25,746 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-04T15:15:25,746 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64157' and parent='/hbase-test1' 2024-12-04T15:15:25,748 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0bc59bbf to 127.0.0.1:64157 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@266ff9a4 2024-12-04T15:15:25,750 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-04T15:15:25,750 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:25,759 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-04T15:15:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=246 2024-12-04T15:15:25,771 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:25,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=246, resume processing ppid=245 2024-12-04T15:15:25,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=246, ppid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-04T15:15:25,773 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:25,773 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:25,794 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:25,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 305 msec 2024-12-04T15:15:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-04T15:15:25,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:25,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:25,840 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:25,841 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=247, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:25,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=247, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-04T15:15:25,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247 2024-12-04T15:15:25,846 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-04T15:15:25,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=248, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:25,849 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:25,850 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=248, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:25,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=248, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-04T15:15:25,851 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248 2024-12-04T15:15:25,857 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:25,912 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=596 (was 590) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c16bf3f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c16bf3f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0bc59bbf-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0bc59bbf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x3c16bf3f-SendThread(127.0.0.1:64157) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64157@0x0bc59bbf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$859/0x00007f7d54be5c00.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=951 (was 943) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1182 (was 1163) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2191 (was 2209) 2024-12-04T15:15:25,912 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2024-12-04T15:15:25,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=596, OpenFileDescriptor=951, MaxFileDescriptor=1048576, SystemLoadAverage=1182, ProcessCount=11, AvailableMemoryMB=2190 2024-12-04T15:15:25,935 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2024-12-04T15:15:25,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-04T15:15:25,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=250, ppid=249, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-04T15:15:26,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=250 2024-12-04T15:15:26,139 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-04T15:15:26,213 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.ReplicationSource(231): queueId=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSource: 1, currentBandwidth=0 2024-12-04T15:15:26,219 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=250 2024-12-04T15:15:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=250 2024-12-04T15:15:26,222 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-c6f2e1d35cf1,37535,1733325265751 (queues=1) is replicating from cluster=6aa3465c-d757-4950-ad0d-9868eb0e70b4 to cluster=6aa3465c-d757-4950-ad0d-9868eb0e70b4 2024-12-04T15:15:26,222 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=c6f2e1d35cf1%2C37535%2C1733325265751 2024-12-04T15:15:26,223 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-c6f2e1d35cf1,37535,1733325265751, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-04T15:15:26,227 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:26,235 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.wal-reader.c6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/WALs/c6f2e1d35cf1,37535,1733325265751/c6f2e1d35cf1%2C37535%2C1733325265751.1733325271247, startPosition=0, beingWritten=true 2024-12-04T15:15:26,236 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: c6f2e1d35cf1%2C37535%2C1733325265751 2024-12-04T15:15:26,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=250, resume processing ppid=249 2024-12-04T15:15:26,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=250, ppid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 243 msec 2024-12-04T15:15:26,243 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-04T15:15:26,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 305 msec 2024-12-04T15:15:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-04T15:15:26,255 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:26,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-04T15:15:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-04T15:15:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-04T15:15:26,265 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-04T15:15:26,267 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=251, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:26,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=251, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=8 msec 2024-12-04T15:15:26,270 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-04T15:15:26,376 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-04T15:15:26,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-04T15:15:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=252, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:26,398 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:26,399 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:26,399 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:26,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-04T15:15:26,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:26,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-04T15:15:26,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-04T15:15:26,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-04T15:15:26,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=253, ppid=252, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-04T15:15:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:26,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37535 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=253 2024-12-04T15:15:26,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-04T15:15:26,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-04T15:15:26,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-c6f2e1d35cf1,37535,1733325265751 because: Replication stream was removed by a user 2024-12-04T15:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:27,719 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-04T15:15:27,720 WARN [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.wal-reader.c6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-04T15:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:28,719 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0.replicationSource,1-c6f2e1d35cf1,37535,1733325265751.replicationSource.shipperc6f2e1d35cf1%2C37535%2C1733325265751,1-c6f2e1d35cf1,37535,1733325265751 terminated 2024-12-04T15:15:28,720 INFO [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(562): Done with the queue 1-c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:28,726 DEBUG [RS_REFRESH_PEER-regionserver/c6f2e1d35cf1:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=253 2024-12-04T15:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(4169): Remote procedure done, pid=253 2024-12-04T15:15:28,733 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on c6f2e1d35cf1,37535,1733325265751 suceeded 2024-12-04T15:15:28,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=253, resume processing ppid=252 2024-12-04T15:15:28,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=253, ppid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1790 sec 2024-12-04T15:15:28,736 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-04T15:15:28,736 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-04T15:15:28,759 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-04T15:15:28,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.3820 sec 2024-12-04T15:15:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-04T15:15:30,546 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-04T15:15:30,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-04T15:15:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] procedure2.ProcedureExecutor(1139): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-04T15:15:30,552 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-04T15:15:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-04T15:15:30,553 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=254, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-04T15:15:30,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=254, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-04T15:15:30,558 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-04T15:15:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46533 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-04T15:15:30,666 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-04T15:15:30,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=598 (was 596) - Thread LEAK? -, OpenFileDescriptor=956 (was 951) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1247 (was 1182) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2134 (was 2190) 2024-12-04T15:15:30,704 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2024-12-04T15:15:30,705 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:15:30,705 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:15:30,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T15:15:30,706 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:15:30,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:15:30,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:15:30,708 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T15:15:30,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T15:15:30,709 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=910324829, stopped=false 2024-12-04T15:15:30,710 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c6f2e1d35cf1,46533,1733325264441 2024-12-04T15:15:30,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T15:15:30,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:15:30,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T15:15:30,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:15:30,731 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:15:30,732 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T15:15:30,732 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:15:30,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,733 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c6f2e1d35cf1,37535,1733325265751' ***** 2024-12-04T15:15:30,733 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T15:15:30,733 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T15:15:30,734 INFO [RS:0;c6f2e1d35cf1:37535 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T15:15:30,734 INFO [RS:0;c6f2e1d35cf1:37535 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T15:15:30,734 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(3091): Received CLOSE for 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:30,734 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T15:15:30,734 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:15:30,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(959): stopping server c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c6f2e1d35cf1:37535. 2024-12-04T15:15:30,747 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T15:15:30,747 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T15:15:30,747 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 169fab88624a99a661f8d8b6fa142c42, disabling compactions & flushes 2024-12-04T15:15:30,747 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T15:15:30,747 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:15:30,747 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:15:30,747 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. after waiting 0 ms 2024-12-04T15:15:30,747 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:15:30,751 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T15:15:30,751 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1325): Online Regions={169fab88624a99a661f8d8b6fa142c42=hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T15:15:30,751 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:30,752 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T15:15:30,752 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T15:15:30,752 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T15:15:30,753 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T15:15:30,753 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T15:15:30,756 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 169fab88624a99a661f8d8b6fa142c42 3/3 column families, dataSize=5.05 KB heapSize=10.72 KB 2024-12-04T15:15:30,756 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-12-04T15:15:30,807 INFO [regionserver/c6f2e1d35cf1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:15:30,900 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/info/b5c0c2d2bbf042f29c20ffb222b40eff is 147, key is hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42./info:regioninfo/1733325274166/Put/seqid=0 2024-12-04T15:15:30,900 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/hfileref/0ddf5af96b5c4a5a84c74f099f44b7b4 is 33, key is 1/hfileref:/1733325328753/DeleteFamily/seqid=0 2024-12-04T15:15:30,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741838_1014 (size=5733) 2024-12-04T15:15:30,954 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:30,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741839_1015 (size=6631) 2024-12-04T15:15:31,030 INFO [regionserver/c6f2e1d35cf1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-12-04T15:15:31,030 INFO [regionserver/c6f2e1d35cf1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T15:15:31,030 INFO [regionserver/c6f2e1d35cf1:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T15:15:31,155 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:31,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T15:15:31,354 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=638 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/hfileref/0ddf5af96b5c4a5a84c74f099f44b7b4 2024-12-04T15:15:31,355 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:31,373 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/info/b5c0c2d2bbf042f29c20ffb222b40eff 2024-12-04T15:15:31,433 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ddf5af96b5c4a5a84c74f099f44b7b4 2024-12-04T15:15:31,518 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/ns/eda746cedcf6480da1b16ca51ca8ba1c is 43, key is default/ns:d/1733325272321/Put/seqid=0 2024-12-04T15:15:31,520 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/queue/c8e25a97d8cd47699d93d283f5195bd0 is 63, key is 1-c6f2e1d35cf1,37535,1733325265751/queue:/1733325328721/DeleteFamily/seqid=0 2024-12-04T15:15:31,555 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:31,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741840_1016 (size=5153) 2024-12-04T15:15:31,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741841_1017 (size=6582) 2024-12-04T15:15:31,755 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:31,956 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:31,979 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/ns/eda746cedcf6480da1b16ca51ca8ba1c 2024-12-04T15:15:31,980 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.43 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/queue/c8e25a97d8cd47699d93d283f5195bd0 2024-12-04T15:15:32,048 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8e25a97d8cd47699d93d283f5195bd0 2024-12-04T15:15:32,055 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/hfileref/0ddf5af96b5c4a5a84c74f099f44b7b4 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/hfileref/0ddf5af96b5c4a5a84c74f099f44b7b4 2024-12-04T15:15:32,101 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/table/55cd965e10dd4e3ba472a2818b3cd4e0 is 53, key is hbase:replication/table:state/1733325274235/Put/seqid=0 2024-12-04T15:15:32,105 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ddf5af96b5c4a5a84c74f099f44b7b4 2024-12-04T15:15:32,106 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/hfileref/0ddf5af96b5c4a5a84c74f099f44b7b4, entries=22, sequenceid=70, filesize=5.6 K 2024-12-04T15:15:32,119 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/.tmp/queue/c8e25a97d8cd47699d93d283f5195bd0 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/queue/c8e25a97d8cd47699d93d283f5195bd0 2024-12-04T15:15:32,144 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8e25a97d8cd47699d93d283f5195bd0 2024-12-04T15:15:32,145 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/queue/c8e25a97d8cd47699d93d283f5195bd0, entries=22, sequenceid=70, filesize=6.4 K 2024-12-04T15:15:32,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741842_1018 (size=5256) 2024-12-04T15:15:32,157 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 169fab88624a99a661f8d8b6fa142c42 2024-12-04T15:15:32,159 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for 169fab88624a99a661f8d8b6fa142c42 in 1401ms, sequenceid=70, compaction requested=false 2024-12-04T15:15:32,217 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/replication/169fab88624a99a661f8d8b6fa142c42/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1 2024-12-04T15:15:32,224 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:15:32,224 INFO [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:15:32,224 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 169fab88624a99a661f8d8b6fa142c42: Waiting for close lock at 1733325330747Running coprocessor pre-close hooks at 1733325330747Disabling compacts and flushes for region at 1733325330747Disabling writes for close at 1733325330747Obtaining lock to block concurrent updates at 1733325330756 (+9 ms)Preparing flush snapshotting stores in 169fab88624a99a661f8d8b6fa142c42 at 1733325330756Finished memstore snapshotting hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42., syncing WAL and waiting on mvcc, flushsize=dataSize=5170, getHeapSize=10928, getOffHeapSize=0, getCellsCount=66 at 1733325330774 (+18 ms)Flushing stores of hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. at 1733325330783 (+9 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/hfileref: creating writer at 1733325330790 (+7 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/hfileref: appending metadata at 1733325330877 (+87 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/hfileref: closing flushed file at 1733325330884 (+7 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/queue: creating writer at 1733325331455 (+571 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/queue: appending metadata at 1733325331514 (+59 ms)Flushing 169fab88624a99a661f8d8b6fa142c42/queue: closing flushed file at 1733325331516 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c443e20: reopening flushed file at 1733325332048 (+532 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@97de490: reopening flushed file at 1733325332109 (+61 ms)Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for 169fab88624a99a661f8d8b6fa142c42 in 1401ms, sequenceid=70, compaction requested=false at 1733325332159 (+50 ms)Writing region close event to WAL at 1733325332195 (+36 ms)Running coprocessor post-close hooks at 1733325332220 (+25 ms)Closed at 1733325332224 (+4 ms) 2024-12-04T15:15:32,225 DEBUG [RS_CLOSE_REGION-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:replication,,1733325273169.169fab88624a99a661f8d8b6fa142c42. 2024-12-04T15:15:32,357 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T15:15:32,357 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T15:15:32,357 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:15:32,554 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/table/55cd965e10dd4e3ba472a2818b3cd4e0 2024-12-04T15:15:32,560 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:15:32,595 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/info/b5c0c2d2bbf042f29c20ffb222b40eff as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/info/b5c0c2d2bbf042f29c20ffb222b40eff 2024-12-04T15:15:32,621 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/info/b5c0c2d2bbf042f29c20ffb222b40eff, entries=10, sequenceid=11, filesize=6.5 K 2024-12-04T15:15:32,623 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/ns/eda746cedcf6480da1b16ca51ca8ba1c as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/ns/eda746cedcf6480da1b16ca51ca8ba1c 2024-12-04T15:15:32,664 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/ns/eda746cedcf6480da1b16ca51ca8ba1c, entries=2, sequenceid=11, filesize=5.0 K 2024-12-04T15:15:32,669 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/.tmp/table/55cd965e10dd4e3ba472a2818b3cd4e0 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/table/55cd965e10dd4e3ba472a2818b3cd4e0 2024-12-04T15:15:32,696 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/table/55cd965e10dd4e3ba472a2818b3cd4e0, entries=2, sequenceid=11, filesize=5.1 K 2024-12-04T15:15:32,697 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1944ms, sequenceid=11, compaction requested=false 2024-12-04T15:15:32,762 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T15:15:32,762 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-04T15:15:32,768 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T15:15:32,768 INFO [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T15:15:32,768 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733325330752Running coprocessor pre-close hooks at 1733325330752Disabling compacts and flushes for region at 1733325330752Disabling writes for close at 1733325330753 (+1 ms)Obtaining lock to block concurrent updates at 1733325330756 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733325330756Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1368, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1733325330773 (+17 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733325330784 (+11 ms)Flushing 1588230740/info: creating writer at 1733325330790 (+6 ms)Flushing 1588230740/info: appending metadata at 1733325330880 (+90 ms)Flushing 1588230740/info: closing flushed file at 1733325330883 (+3 ms)Flushing 1588230740/ns: creating writer at 1733325331454 (+571 ms)Flushing 1588230740/ns: appending metadata at 1733325331516 (+62 ms)Flushing 1588230740/ns: closing flushed file at 1733325331516Flushing 1588230740/table: creating writer at 1733325332014 (+498 ms)Flushing 1588230740/table: appending metadata at 1733325332078 (+64 ms)Flushing 1588230740/table: closing flushed file at 1733325332078Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1938f5f6: reopening flushed file at 1733325332589 (+511 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2682111d: reopening flushed file at 1733325332621 (+32 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f50e8d4: reopening flushed file at 1733325332664 (+43 ms)Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1944ms, sequenceid=11, compaction requested=false at 1733325332697 (+33 ms)Writing region close event to WAL at 1733325332729 (+32 ms)Running coprocessor post-close hooks at 1733325332768 (+39 ms)Closed at 1733325332768 2024-12-04T15:15:32,768 DEBUG [RS_CLOSE_META-regionserver/c6f2e1d35cf1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T15:15:32,967 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(976): stopping server c6f2e1d35cf1,37535,1733325265751; all regions closed. 2024-12-04T15:15:32,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741834_1010 (size=2742) 2024-12-04T15:15:33,003 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs 2024-12-04T15:15:33,003 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6f2e1d35cf1%2C37535%2C1733325265751.meta:.meta(num 1733325271804) 2024-12-04T15:15:33,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741837_1013 (size=11128) 2024-12-04T15:15:33,038 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs 2024-12-04T15:15:33,038 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6f2e1d35cf1%2C37535%2C1733325265751.rep:(num 1733325273894) 2024-12-04T15:15:33,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741833_1009 (size=93) 2024-12-04T15:15:33,105 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/oldWALs 2024-12-04T15:15:33,106 INFO [RS:0;c6f2e1d35cf1:37535 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c6f2e1d35cf1%2C37535%2C1733325265751:(num 1733325271247) 2024-12-04T15:15:33,106 DEBUG [RS:0;c6f2e1d35cf1:37535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T15:15:33,106 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T15:15:33,106 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:15:33,106 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.ChoreService(370): Chore service for: regionserver/c6f2e1d35cf1:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T15:15:33,107 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:15:33,107 INFO [regionserver/c6f2e1d35cf1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:15:33,108 INFO [RS:0;c6f2e1d35cf1:37535 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37535 2024-12-04T15:15:33,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c6f2e1d35cf1,37535,1733325265751 2024-12-04T15:15:33,189 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:15:33,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T15:15:33,202 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c6f2e1d35cf1,37535,1733325265751] 2024-12-04T15:15:33,211 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c6f2e1d35cf1,37535,1733325265751 already deleted, retry=false 2024-12-04T15:15:33,211 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c6f2e1d35cf1,37535,1733325265751 expired; onlineServers=0 2024-12-04T15:15:33,211 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c6f2e1d35cf1,46533,1733325264441' ***** 2024-12-04T15:15:33,211 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T15:15:33,211 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T15:15:33,212 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T15:15:33,212 DEBUG [M:0;c6f2e1d35cf1:46533 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T15:15:33,212 DEBUG [M:0;c6f2e1d35cf1:46533 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T15:15:33,212 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.ChoreService(370): Chore service for: master/c6f2e1d35cf1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T15:15:33,212 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T15:15:33,212 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.large.0-1733325270589 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.large.0-1733325270589,5,FailOnTimeoutGroup] 2024-12-04T15:15:33,212 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T15:15:33,213 DEBUG [M:0;c6f2e1d35cf1:46533 {}] master.HMaster(1795): Stopping service threads 2024-12-04T15:15:33,213 INFO [M:0;c6f2e1d35cf1:46533 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T15:15:33,213 INFO [M:0;c6f2e1d35cf1:46533 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T15:15:33,213 INFO [M:0;c6f2e1d35cf1:46533 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T15:15:33,214 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T15:15:33,214 DEBUG [master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.small.0-1733325270595 {}] cleaner.HFileCleaner(306): Exit Thread[master/c6f2e1d35cf1:0:becomeActiveMaster-HFileCleaner.small.0-1733325270595,5,FailOnTimeoutGroup] 2024-12-04T15:15:33,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T15:15:33,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T15:15:33,230 DEBUG [M:0;c6f2e1d35cf1:46533 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-04T15:15:33,231 DEBUG [M:0;c6f2e1d35cf1:46533 {}] master.ActiveMasterManager(353): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-04T15:15:33,235 INFO [M:0;c6f2e1d35cf1:46533 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/.lastflushedseqids 2024-12-04T15:15:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741843_1019 (size=172) 2024-12-04T15:15:33,290 INFO [M:0;c6f2e1d35cf1:46533 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T15:15:33,290 INFO [M:0;c6f2e1d35cf1:46533 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T15:15:33,291 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T15:15:33,291 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:15:33,291 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:15:33,291 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T15:15:33,291 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:15:33,291 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=753.47 KB heapSize=901.72 KB 2024-12-04T15:15:33,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:15:33,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37535-0x101a21658430001, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:15:33,300 INFO [RS:0;c6f2e1d35cf1:37535 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:15:33,300 INFO [RS:0;c6f2e1d35cf1:37535 {}] regionserver.HRegionServer(1031): Exiting; stopping=c6f2e1d35cf1,37535,1733325265751; zookeeper connection closed. 2024-12-04T15:15:33,302 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53e18261 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53e18261 2024-12-04T15:15:33,303 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T15:15:33,342 DEBUG [M:0;c6f2e1d35cf1:46533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/334ae40fa9f944df8a0e2ae94f2f2066 is 82, key is hbase:meta,,1/info:regioninfo/1733325272122/Put/seqid=0 2024-12-04T15:15:33,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741844_1020 (size=5672) 2024-12-04T15:15:33,370 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1805 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/334ae40fa9f944df8a0e2ae94f2f2066 2024-12-04T15:15:33,501 DEBUG [M:0;c6f2e1d35cf1:46533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/82f2882c37fa4a279ec75e28127dce98 is 2004, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1733325309549/Put/seqid=0 2024-12-04T15:15:33,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741845_1021 (size=163383) 2024-12-04T15:15:33,539 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.91 KB at sequenceid=1805 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/82f2882c37fa4a279ec75e28127dce98 2024-12-04T15:15:33,560 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82f2882c37fa4a279ec75e28127dce98 2024-12-04T15:15:33,607 DEBUG [M:0;c6f2e1d35cf1:46533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b55e36e009a45dc8c6226a6c63338c5 is 69, key is c6f2e1d35cf1,37535,1733325265751/rs:state/1733325270715/Put/seqid=0 2024-12-04T15:15:33,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741846_1022 (size=5156) 2024-12-04T15:15:33,663 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1805 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b55e36e009a45dc8c6226a6c63338c5 2024-12-04T15:15:33,733 DEBUG [M:0;c6f2e1d35cf1:46533 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adabe0459974a8ea9b9f76cf066ace9 is 68, key is replication_peer_modification_on/state:d/1733325330670/Put/seqid=0 2024-12-04T15:15:33,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741847_1023 (size=5154) 2024-12-04T15:15:34,206 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1805 (bloomFilter=true), to=hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adabe0459974a8ea9b9f76cf066ace9 2024-12-04T15:15:34,239 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/334ae40fa9f944df8a0e2ae94f2f2066 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/334ae40fa9f944df8a0e2ae94f2f2066 2024-12-04T15:15:34,261 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/334ae40fa9f944df8a0e2ae94f2f2066, entries=8, sequenceid=1805, filesize=5.5 K 2024-12-04T15:15:34,263 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/82f2882c37fa4a279ec75e28127dce98 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/82f2882c37fa4a279ec75e28127dce98 2024-12-04T15:15:34,281 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 82f2882c37fa4a279ec75e28127dce98 2024-12-04T15:15:34,282 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/82f2882c37fa4a279ec75e28127dce98, entries=254, sequenceid=1805, filesize=159.6 K 2024-12-04T15:15:34,286 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4b55e36e009a45dc8c6226a6c63338c5 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4b55e36e009a45dc8c6226a6c63338c5 2024-12-04T15:15:34,303 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4b55e36e009a45dc8c6226a6c63338c5, entries=1, sequenceid=1805, filesize=5.0 K 2024-12-04T15:15:34,305 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adabe0459974a8ea9b9f76cf066ace9 as hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1adabe0459974a8ea9b9f76cf066ace9 2024-12-04T15:15:34,342 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41653/user/jenkins/test-data/5db757f2-f9f3-5ed2-7f75-4154d8b60bcb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1adabe0459974a8ea9b9f76cf066ace9, entries=1, sequenceid=1805, filesize=5.0 K 2024-12-04T15:15:34,363 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(3140): Finished flush of dataSize ~753.47 KB/771551, heapSize ~901.66 KB/923296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1071ms, sequenceid=1805, compaction requested=false 2024-12-04T15:15:34,430 INFO [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T15:15:34,431 DEBUG [M:0;c6f2e1d35cf1:46533 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733325333291Disabling compacts and flushes for region at 1733325333291Disabling writes for close at 1733325333291Obtaining lock to block concurrent updates at 1733325333291Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733325333291Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=771551, getHeapSize=923296, getOffHeapSize=0, getCellsCount=1996 at 1733325333292 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733325333302 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733325333303 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733325333341 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733325333342 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733325333391 (+49 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733325333498 (+107 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733325333498Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733325333561 (+63 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733325333606 (+45 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733325333606Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733325333687 (+81 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733325333732 (+45 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733325333732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@511917c5: reopening flushed file at 1733325334230 (+498 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21a75e83: reopening flushed file at 1733325334261 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7890fbd0: reopening flushed file at 1733325334282 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e62bb6b: reopening flushed file at 1733325334303 (+21 ms)Finished flush of dataSize ~753.47 KB/771551, heapSize ~901.66 KB/923296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1071ms, sequenceid=1805, compaction requested=false at 1733325334363 (+60 ms)Writing region close event to WAL at 1733325334430 (+67 ms)Closed at 1733325334430 2024-12-04T15:15:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40245 is added to blk_1073741830_1006 (size=890093) 2024-12-04T15:15:34,482 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T15:15:34,483 INFO [M:0;c6f2e1d35cf1:46533 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T15:15:34,486 INFO [M:0;c6f2e1d35cf1:46533 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46533 2024-12-04T15:15:34,486 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T15:15:34,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:15:34,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46533-0x101a21658430000, quorum=127.0.0.1:64157, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T15:15:34,658 INFO [M:0;c6f2e1d35cf1:46533 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T15:15:34,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e80f2cb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T15:15:34,690 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:15:34,691 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:15:34,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62b38b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:15:34,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@698dfd8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.log.dir/,STOPPED} 2024-12-04T15:15:34,700 WARN [BP-1046799081-172.17.0.3-1733325260798 heartbeating to localhost/127.0.0.1:41653 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T15:15:34,701 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T15:15:34,701 WARN [BP-1046799081-172.17.0.3-1733325260798 heartbeating to localhost/127.0.0.1:41653 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1046799081-172.17.0.3-1733325260798 (Datanode Uuid 8f9716f7-691d-4710-be47-7687cc4aaa4b) service to localhost/127.0.0.1:41653 2024-12-04T15:15:34,701 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T15:15:34,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/data/data1/current/BP-1046799081-172.17.0.3-1733325260798 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:15:34,705 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/cluster_ae8aa94e-5e2e-740f-9987-4ed871f7563a/data/data2/current/BP-1046799081-172.17.0.3-1733325260798 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T15:15:34,705 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T15:15:34,755 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1441d181{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T15:15:34,764 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T15:15:34,765 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T15:15:34,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T15:15:34,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/414ba69a-7da7-6850-de37-9f6932a63575/hadoop.log.dir/,STOPPED} 2024-12-04T15:15:34,812 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T15:15:34,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down