2024-12-03 02:23:38,405 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-03 02:23:38,419 main DEBUG Took 0.012110 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 02:23:38,420 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 02:23:38,420 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 02:23:38,422 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 02:23:38,423 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,450 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 02:23:38,465 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,469 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,470 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,471 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,473 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,475 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,476 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,477 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,477 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 02:23:38,479 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,480 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 02:23:38,481 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 02:23:38,483 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 02:23:38,485 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 02:23:38,486 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 02:23:38,488 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 02:23:38,488 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 02:23:38,498 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 02:23:38,501 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 02:23:38,503 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 02:23:38,504 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 02:23:38,504 main DEBUG createAppenders(={Console}) 2024-12-03 02:23:38,505 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec initialized 2024-12-03 02:23:38,505 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-03 02:23:38,506 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec OK. 2024-12-03 02:23:38,507 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 02:23:38,507 main DEBUG OutputStream closed 2024-12-03 02:23:38,507 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 02:23:38,508 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 02:23:38,508 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4efc180e OK 2024-12-03 02:23:38,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 02:23:38,612 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 02:23:38,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 02:23:38,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 02:23:38,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 02:23:38,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 02:23:38,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 02:23:38,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 02:23:38,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 02:23:38,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 02:23:38,618 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 02:23:38,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 02:23:38,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 02:23:38,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 02:23:38,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 02:23:38,620 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 02:23:38,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 02:23:38,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 02:23:38,624 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 02:23:38,624 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-12-03 02:23:38,624 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 02:23:38,626 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-12-03T02:23:38,906 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391 2024-12-03 02:23:38,910 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 02:23:38,910 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T02:23:38,947 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-03T02:23:38,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T02:23:39,005 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587, deleteOnExit=true 2024-12-03T02:23:39,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T02:23:39,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/test.cache.data in system properties and HBase conf 2024-12-03T02:23:39,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T02:23:39,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.log.dir in system properties and HBase conf 2024-12-03T02:23:39,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T02:23:39,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T02:23:39,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T02:23:39,088 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T02:23:39,172 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T02:23:39,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:23:39,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T02:23:39,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T02:23:39,178 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:23:39,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T02:23:39,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T02:23:39,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T02:23:39,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:23:39,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T02:23:39,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/nfs.dump.dir in system properties and HBase conf 2024-12-03T02:23:39,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/java.io.tmpdir in system properties and HBase conf 2024-12-03T02:23:39,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T02:23:39,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T02:23:39,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T02:23:40,209 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T02:23:40,274 INFO [Time-limited test {}] log.Log(170): Logging initialized @2571ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T02:23:40,343 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:23:40,416 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:23:40,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:23:40,450 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:23:40,452 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T02:23:40,466 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:23:40,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:23:40,471 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:23:40,665 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1441d181{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/java.io.tmpdir/jetty-localhost-34551-hadoop-hdfs-3_4_1-tests_jar-_-any-10925674086374977801/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:23:40,685 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:34551} 2024-12-03T02:23:40,686 INFO [Time-limited test {}] server.Server(415): Started @2984ms 2024-12-03T02:23:41,187 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T02:23:41,194 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T02:23:41,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T02:23:41,198 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T02:23:41,198 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T02:23:41,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@698dfd8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.log.dir/,AVAILABLE} 2024-12-03T02:23:41,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62b38b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T02:23:41,308 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e80f2cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/java.io.tmpdir/jetty-localhost-42291-hadoop-hdfs-3_4_1-tests_jar-_-any-5831330851709207899/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:23:41,309 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:42291} 2024-12-03T02:23:41,310 INFO [Time-limited test {}] server.Server(415): Started @3608ms 2024-12-03T02:23:41,381 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T02:23:42,698 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/data/data2/current/BP-1640616088-172.17.0.3-1733192619772/current, will proceed with Du for space computation calculation, 2024-12-03T02:23:42,698 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/data/data1/current/BP-1640616088-172.17.0.3-1733192619772/current, will proceed with Du for space computation calculation, 2024-12-03T02:23:42,733 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T02:23:42,784 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76c1010ad026861d with lease ID 0x8e0ff7c45743c855: Processing first storage report for DS-29454614-db68-4ec7-803d-98844b639c75 from datanode DatanodeRegistration(127.0.0.1:33433, datanodeUuid=c613eedb-8ecc-4dc8-9c59-b16125dd16b5, infoPort=37577, infoSecurePort=0, ipcPort=46401, storageInfo=lv=-57;cid=testClusterID;nsid=348881602;c=1733192619772) 2024-12-03T02:23:42,785 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76c1010ad026861d with lease ID 0x8e0ff7c45743c855: from storage DS-29454614-db68-4ec7-803d-98844b639c75 node DatanodeRegistration(127.0.0.1:33433, datanodeUuid=c613eedb-8ecc-4dc8-9c59-b16125dd16b5, infoPort=37577, infoSecurePort=0, ipcPort=46401, storageInfo=lv=-57;cid=testClusterID;nsid=348881602;c=1733192619772), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T02:23:42,785 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76c1010ad026861d with lease ID 0x8e0ff7c45743c855: Processing first storage report for DS-1b5e30e2-9f36-4b2d-82c9-d0c995734cdc from datanode DatanodeRegistration(127.0.0.1:33433, datanodeUuid=c613eedb-8ecc-4dc8-9c59-b16125dd16b5, infoPort=37577, infoSecurePort=0, ipcPort=46401, storageInfo=lv=-57;cid=testClusterID;nsid=348881602;c=1733192619772) 2024-12-03T02:23:42,786 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76c1010ad026861d with lease ID 0x8e0ff7c45743c855: from storage DS-1b5e30e2-9f36-4b2d-82c9-d0c995734cdc node DatanodeRegistration(127.0.0.1:33433, datanodeUuid=c613eedb-8ecc-4dc8-9c59-b16125dd16b5, infoPort=37577, infoSecurePort=0, ipcPort=46401, storageInfo=lv=-57;cid=testClusterID;nsid=348881602;c=1733192619772), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T02:23:42,792 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391 2024-12-03T02:23:42,876 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/zookeeper_0, clientPort=64747, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T02:23:42,887 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64747 2024-12-03T02:23:42,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:42,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741825_1001 (size=7) 2024-12-03T02:23:43,549 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731 with version=8 2024-12-03T02:23:43,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/hbase-staging 2024-12-03T02:23:43,636 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T02:23:43,854 INFO [Time-limited test {}] client.ConnectionUtils(128): master/64035cbb0b46:0 server-side Connection retries=6 2024-12-03T02:23:43,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:43,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:43,866 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:23:43,866 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:43,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:23:44,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T02:23:44,070 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T02:23:44,080 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T02:23:44,085 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:23:44,111 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 60470 (auto-detected) 2024-12-03T02:23:44,113 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T02:23:44,131 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39895 2024-12-03T02:23:44,160 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39895 connecting to ZooKeeper ensemble=127.0.0.1:64747 2024-12-03T02:23:44,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398950x0, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:23:44,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39895-0x1019a4a54f20000 connected 2024-12-03T02:23:44,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:44,331 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:44,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:23:44,348 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731, hbase.cluster.distributed=false 2024-12-03T02:23:44,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:23:44,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39895 2024-12-03T02:23:44,379 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39895 2024-12-03T02:23:44,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39895 2024-12-03T02:23:44,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39895 2024-12-03T02:23:44,387 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39895 2024-12-03T02:23:44,523 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/64035cbb0b46:0 server-side Connection retries=6 2024-12-03T02:23:44,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:44,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:44,526 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T02:23:44,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T02:23:44,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T02:23:44,531 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T02:23:44,536 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T02:23:44,537 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40081 2024-12-03T02:23:44,539 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40081 connecting to ZooKeeper ensemble=127.0.0.1:64747 2024-12-03T02:23:44,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:44,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:44,559 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400810x0, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T02:23:44,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:400810x0, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:23:44,561 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40081-0x1019a4a54f20001 connected 2024-12-03T02:23:44,567 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T02:23:44,580 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T02:23:44,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T02:23:44,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T02:23:44,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40081 2024-12-03T02:23:44,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40081 2024-12-03T02:23:44,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40081 2024-12-03T02:23:44,590 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40081 2024-12-03T02:23:44,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40081 2024-12-03T02:23:44,606 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;64035cbb0b46:39895 2024-12-03T02:23:44,607 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/64035cbb0b46,39895,1733192623709 2024-12-03T02:23:44,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:23:44,627 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:23:44,637 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/64035cbb0b46,39895,1733192623709 2024-12-03T02:23:44,668 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T02:23:44,668 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:44,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:44,669 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T02:23:44,673 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/64035cbb0b46,39895,1733192623709 from backup master directory 2024-12-03T02:23:44,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/64035cbb0b46,39895,1733192623709 2024-12-03T02:23:44,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:23:44,688 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T02:23:44,689 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:23:44,689 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=64035cbb0b46,39895,1733192623709 2024-12-03T02:23:44,692 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T02:23:44,695 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T02:23:44,748 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/hbase.id] with ID: 5b5d6b8d-1904-4383-be12-bce3422a64f5 2024-12-03T02:23:44,748 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/.tmp/hbase.id 2024-12-03T02:23:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741826_1002 (size=42) 2024-12-03T02:23:44,759 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/.tmp/hbase.id]:[hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/hbase.id] 2024-12-03T02:23:44,807 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T02:23:44,812 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T02:23:44,832 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-03T02:23:44,842 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:44,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:44,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741827_1003 (size=196) 2024-12-03T02:23:44,875 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:23:44,877 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T02:23:44,891 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:44,895 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:23:44,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741828_1004 (size=1189) 2024-12-03T02:23:45,336 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store 2024-12-03T02:23:45,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741829_1005 (size=34) 2024-12-03T02:23:45,362 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T02:23:45,365 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:45,366 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:23:45,366 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:23:45,366 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:23:45,367 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T02:23:45,367 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:23:45,367 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:23:45,368 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733192625366Disabling compacts and flushes for region at 1733192625366Disabling writes for close at 1733192625367 (+1 ms)Writing region close event to WAL at 1733192625367Closed at 1733192625367 2024-12-03T02:23:45,370 WARN [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/.initializing 2024-12-03T02:23:45,371 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/WALs/64035cbb0b46,39895,1733192623709 2024-12-03T02:23:45,378 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:23:45,391 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C39895%2C1733192623709, suffix=, logDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/WALs/64035cbb0b46,39895,1733192623709, archiveDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/oldWALs, maxLogs=10 2024-12-03T02:23:45,411 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/WALs/64035cbb0b46,39895,1733192623709/64035cbb0b46%2C39895%2C1733192623709.1733192625395, exclude list is [], retry=0 2024-12-03T02:23:45,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33433,DS-29454614-db68-4ec7-803d-98844b639c75,DISK] 2024-12-03T02:23:45,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T02:23:45,460 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/WALs/64035cbb0b46,39895,1733192623709/64035cbb0b46%2C39895%2C1733192623709.1733192625395 2024-12-03T02:23:45,461 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37577:37577)] 2024-12-03T02:23:45,461 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:23:45,462 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:45,465 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,466 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T02:23:45,537 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:45,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:45,541 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T02:23:45,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:45,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:45,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T02:23:45,550 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:45,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:45,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T02:23:45,554 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:45,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:45,556 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,560 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,561 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,567 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,568 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,572 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:23:45,577 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T02:23:45,582 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:23:45,584 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64412963, jitterRate=-0.04017205536365509}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:23:45,590 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733192625479Initializing all the Stores at 1733192625481 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192625482 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192625482Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192625483 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192625483Cleaning up temporary data from old regions at 1733192625568 (+85 ms)Region opened successfully at 1733192625589 (+21 ms) 2024-12-03T02:23:45,591 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T02:23:45,624 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e144b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:23:45,653 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T02:23:45,663 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T02:23:45,663 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T02:23:45,665 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T02:23:45,667 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T02:23:45,673 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T02:23:45,673 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T02:23:45,699 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T02:23:45,709 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T02:23:45,734 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T02:23:45,738 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T02:23:45,740 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T02:23:45,751 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T02:23:45,753 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T02:23:45,756 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T02:23:45,767 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T02:23:45,769 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T02:23:45,775 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T02:23:45,797 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T02:23:45,808 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T02:23:45,817 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:23:45,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T02:23:45,818 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:45,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:45,821 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=64035cbb0b46,39895,1733192623709, sessionid=0x1019a4a54f20000, setting cluster-up flag (Was=false) 2024-12-03T02:23:45,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:45,851 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:45,975 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T02:23:45,977 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,39895,1733192623709 2024-12-03T02:23:46,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:46,056 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:46,226 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T02:23:46,231 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=64035cbb0b46,39895,1733192623709 2024-12-03T02:23:46,241 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T02:23:46,296 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(746): ClusterId : 5b5d6b8d-1904-4383-be12-bce3422a64f5 2024-12-03T02:23:46,299 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T02:23:46,310 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T02:23:46,319 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T02:23:46,324 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T02:23:46,329 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 64035cbb0b46,39895,1733192623709 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T02:23:46,344 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T02:23:46,344 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T02:23:46,345 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:23:46,346 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:23:46,346 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:23:46,346 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/64035cbb0b46:0, corePoolSize=5, maxPoolSize=5 2024-12-03T02:23:46,346 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/64035cbb0b46:0, corePoolSize=10, maxPoolSize=10 2024-12-03T02:23:46,347 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,347 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:23:46,347 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,350 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733192656350 2024-12-03T02:23:46,352 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T02:23:46,352 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T02:23:46,353 DEBUG [RS:0;64035cbb0b46:40081 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361d866a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=64035cbb0b46/172.17.0.3:0 2024-12-03T02:23:46,354 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T02:23:46,353 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:23:46,354 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T02:23:46,357 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T02:23:46,358 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T02:23:46,359 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T02:23:46,359 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T02:23:46,360 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,362 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:46,362 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T02:23:46,371 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;64035cbb0b46:40081 2024-12-03T02:23:46,371 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T02:23:46,372 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T02:23:46,373 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T02:23:46,374 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T02:23:46,375 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T02:23:46,375 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T02:23:46,375 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T02:23:46,375 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T02:23:46,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741831_1007 (size=1321) 2024-12-03T02:23:46,377 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192626377,5,FailOnTimeoutGroup] 2024-12-03T02:23:46,378 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(2659): reportForDuty to master=64035cbb0b46,39895,1733192623709 with port=40081, startcode=1733192624478 2024-12-03T02:23:46,379 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192626378,5,FailOnTimeoutGroup] 2024-12-03T02:23:46,379 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,379 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T02:23:46,380 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,381 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,388 DEBUG [RS:0;64035cbb0b46:40081 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T02:23:46,419 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53923, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T02:23:46,426 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39895 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39895 {}] master.ServerManager(517): Registering regionserver=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,441 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731 2024-12-03T02:23:46,441 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44743 2024-12-03T02:23:46,442 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T02:23:46,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:23:46,463 DEBUG [RS:0;64035cbb0b46:40081 {}] zookeeper.ZKUtil(111): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,463 WARN [RS:0;64035cbb0b46:40081 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T02:23:46,464 INFO [RS:0;64035cbb0b46:40081 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:23:46,464 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [64035cbb0b46,40081,1733192624478] 2024-12-03T02:23:46,491 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T02:23:46,503 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T02:23:46,508 INFO [RS:0;64035cbb0b46:40081 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T02:23:46,508 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,509 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T02:23:46,514 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T02:23:46,515 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,515 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,515 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/64035cbb0b46:0, corePoolSize=2, maxPoolSize=2 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,516 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,517 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,517 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/64035cbb0b46:0, corePoolSize=1, maxPoolSize=1 2024-12-03T02:23:46,517 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:23:46,517 DEBUG [RS:0;64035cbb0b46:40081 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/64035cbb0b46:0, corePoolSize=3, maxPoolSize=3 2024-12-03T02:23:46,518 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,518 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,518 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,519 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,519 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,519 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,40081,1733192624478-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:23:46,535 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T02:23:46,536 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,40081,1733192624478-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,537 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,537 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.Replication(171): 64035cbb0b46,40081,1733192624478 started 2024-12-03T02:23:46,552 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:46,553 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1482): Serving as 64035cbb0b46,40081,1733192624478, RpcServer on 64035cbb0b46/172.17.0.3:40081, sessionid=0x1019a4a54f20001 2024-12-03T02:23:46,553 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T02:23:46,553 DEBUG [RS:0;64035cbb0b46:40081 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,554 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,40081,1733192624478' 2024-12-03T02:23:46,554 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T02:23:46,555 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T02:23:46,556 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T02:23:46,556 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T02:23:46,556 DEBUG [RS:0;64035cbb0b46:40081 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 64035cbb0b46,40081,1733192624478 2024-12-03T02:23:46,556 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '64035cbb0b46,40081,1733192624478' 2024-12-03T02:23:46,556 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T02:23:46,557 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T02:23:46,557 DEBUG [RS:0;64035cbb0b46:40081 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T02:23:46,558 INFO [RS:0;64035cbb0b46:40081 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T02:23:46,558 INFO [RS:0;64035cbb0b46:40081 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T02:23:46,662 INFO [RS:0;64035cbb0b46:40081 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T02:23:46,666 INFO [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C40081%2C1733192624478, suffix=, logDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478, archiveDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs, maxLogs=32 2024-12-03T02:23:46,682 DEBUG [RS:0;64035cbb0b46:40081 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.1733192626668, exclude list is [], retry=0 2024-12-03T02:23:46,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33433,DS-29454614-db68-4ec7-803d-98844b639c75,DISK] 2024-12-03T02:23:46,693 INFO [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.1733192626668 2024-12-03T02:23:46,695 DEBUG [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37577:37577)] 2024-12-03T02:23:46,780 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T02:23:46,781 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731 2024-12-03T02:23:46,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741833_1009 (size=32) 2024-12-03T02:23:47,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:47,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:23:47,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:23:47,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:23:47,211 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:23:47,211 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:23:47,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:23:47,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:23:47,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:23:47,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:23:47,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740 2024-12-03T02:23:47,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740 2024-12-03T02:23:47,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:23:47,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:23:47,233 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:23:47,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:23:47,242 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:23:47,243 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73518282, jitterRate=0.09550777077674866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:23:47,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733192627196Initializing all the Stores at 1733192627201 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627201Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627202 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192627202Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627202Cleaning up temporary data from old regions at 1733192627232 (+30 ms)Region opened successfully at 1733192627245 (+13 ms) 2024-12-03T02:23:47,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:23:47,245 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:23:47,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:23:47,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:23:47,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:23:47,247 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:23:47,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733192627245Disabling compacts and flushes for region at 1733192627245Disabling writes for close at 1733192627246 (+1 ms)Writing region close event to WAL at 1733192627247 (+1 ms)Closed at 1733192627247 2024-12-03T02:23:47,252 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:23:47,252 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T02:23:47,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T02:23:47,271 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:23:47,274 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T02:23:47,431 DEBUG [64035cbb0b46:39895 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T02:23:47,443 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:47,450 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,40081,1733192624478, state=OPENING 2024-12-03T02:23:47,459 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T02:23:47,467 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:47,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:47,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:23:47,468 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:23:47,470 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T02:23:47,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,40081,1733192624478}] 2024-12-03T02:23:47,646 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T02:23:47,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T02:23:47,662 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T02:23:47,663 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:23:47,663 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T02:23:47,668 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C40081%2C1733192624478.meta, suffix=.meta, logDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478, archiveDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs, maxLogs=32 2024-12-03T02:23:47,690 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.meta.1733192627670.meta, exclude list is [], retry=0 2024-12-03T02:23:47,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33433,DS-29454614-db68-4ec7-803d-98844b639c75,DISK] 2024-12-03T02:23:47,699 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.meta.1733192627670.meta 2024-12-03T02:23:47,700 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37577:37577)] 2024-12-03T02:23:47,700 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:23:47,703 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T02:23:47,706 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T02:23:47,711 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T02:23:47,717 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T02:23:47,718 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:47,719 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T02:23:47,719 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T02:23:47,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T02:23:47,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T02:23:47,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T02:23:47,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T02:23:47,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T02:23:47,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T02:23:47,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,740 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T02:23:47,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T02:23:47,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:47,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T02:23:47,745 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T02:23:47,748 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740 2024-12-03T02:23:47,752 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740 2024-12-03T02:23:47,755 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T02:23:47,756 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T02:23:47,757 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T02:23:47,761 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T02:23:47,764 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61035116, jitterRate=-0.09050589799880981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T02:23:47,765 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T02:23:47,767 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733192627719Writing region info on filesystem at 1733192627720 (+1 ms)Initializing all the Stores at 1733192627723 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627723Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192627723Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733192627723Cleaning up temporary data from old regions at 1733192627756 (+33 ms)Running coprocessor post-open hooks at 1733192627765 (+9 ms)Region opened successfully at 1733192627767 (+2 ms) 2024-12-03T02:23:47,777 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733192627637 2024-12-03T02:23:47,793 DEBUG [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T02:23:47,793 INFO [RS_OPEN_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T02:23:47,795 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:47,798 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 64035cbb0b46,40081,1733192624478, state=OPEN 2024-12-03T02:23:47,856 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:23:47,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T02:23:47,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:23:47,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T02:23:47,857 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:47,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T02:23:47,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=64035cbb0b46,40081,1733192624478 in 385 msec 2024-12-03T02:23:47,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T02:23:47,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec 2024-12-03T02:23:47,875 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T02:23:47,875 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T02:23:47,899 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:23:47,901 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,40081,1733192624478, seqNum=-1] 2024-12-03T02:23:47,926 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:23:47,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:23:47,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6780 sec 2024-12-03T02:23:47,951 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733192627951, completionTime=-1 2024-12-03T02:23:47,953 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T02:23:47,954 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T02:23:47,981 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T02:23:47,981 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733192687981 2024-12-03T02:23:47,981 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733192747981 2024-12-03T02:23:47,982 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-03T02:23:47,984 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,985 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,985 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,986 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-64035cbb0b46:39895, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,987 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,988 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:47,993 DEBUG [master/64035cbb0b46:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T02:23:48,016 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.325sec 2024-12-03T02:23:48,017 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T02:23:48,018 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T02:23:48,020 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T02:23:48,020 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T02:23:48,020 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T02:23:48,021 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T02:23:48,022 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T02:23:48,031 DEBUG [master/64035cbb0b46:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T02:23:48,032 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T02:23:48,032 INFO [master/64035cbb0b46:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=64035cbb0b46,39895,1733192623709-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T02:23:48,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53524f7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,112 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T02:23:48,112 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T02:23:48,116 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,39895,-1 for getting cluster id 2024-12-03T02:23:48,119 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:23:48,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5b5d6b8d-1904-4383-be12-bce3422a64f5' 2024-12-03T02:23:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:23:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5b5d6b8d-1904-4383-be12-bce3422a64f5" 2024-12-03T02:23:48,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2e531e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,39895,-1] 2024-12-03T02:23:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:23:48,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:23:48,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:23:48,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f4d3db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,146 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:23:48,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,40081,1733192624478, seqNum=-1] 2024-12-03T02:23:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:23:48,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T02:23:48,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=64035cbb0b46,39895,1733192623709 2024-12-03T02:23:48,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692e0dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 64035cbb0b46,39895,-1 for getting cluster id 2024-12-03T02:23:48,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T02:23:48,229 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5b5d6b8d-1904-4383-be12-bce3422a64f5' 2024-12-03T02:23:48,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T02:23:48,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5b5d6b8d-1904-4383-be12-bce3422a64f5" 2024-12-03T02:23:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6120f08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [64035cbb0b46,39895,-1] 2024-12-03T02:23:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T02:23:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:23:48,234 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T02:23:48,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b4e4311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T02:23:48,252 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=551, ProcessCount=11, AvailableMemoryMB=2616 2024-12-03T02:23:48,266 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T02:23:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 64035cbb0b46,39895,1733192623709 2024-12-03T02:23:48,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27d3e318 2024-12-03T02:23:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T02:23:48,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T02:23:48,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:48,299 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:48,304 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0af678e0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31fac91d 2024-12-03T02:23:48,304 WARN [PEWorker-3 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-03T02:23:48,359 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0af678e0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0af678e0 to 127.0.0.1:64747 2024-12-03T02:23:48,359 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:48,361 INFO [PEWorker-3 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T02:23:48,366 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication 2024-12-03T02:23:48,370 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T02:23:48,371 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:48,374 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T02:23:48,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741835_1011 (size=1138) 2024-12-03T02:23:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:48,420 DEBUG [PEWorker-3 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication 2024-12-03T02:23:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:48,788 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => badaa07e05f1c98a805a3234eb1add78, NAME => 'hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731 2024-12-03T02:23:48,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741836_1012 (size=44) 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing badaa07e05f1c98a805a3234eb1add78, disabling compactions & flushes 2024-12-03T02:23:48,798 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. after waiting 0 ms 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:48,798 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:48,798 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for badaa07e05f1c98a805a3234eb1add78: Waiting for close lock at 1733192628798Disabling compacts and flushes for region at 1733192628798Disabling writes for close at 1733192628798Writing region close event to WAL at 1733192628798Closed at 1733192628798 2024-12-03T02:23:48,801 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T02:23:48,806 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1733192628801"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733192628801"}]},"ts":"1733192628801"} 2024-12-03T02:23:48,810 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T02:23:48,813 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T02:23:48,816 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733192628813"}]},"ts":"1733192628813"} 2024-12-03T02:23:48,821 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta 2024-12-03T02:23:48,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=badaa07e05f1c98a805a3234eb1add78, ASSIGN}] 2024-12-03T02:23:48,825 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=badaa07e05f1c98a805a3234eb1add78, ASSIGN 2024-12-03T02:23:48,827 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=badaa07e05f1c98a805a3234eb1add78, ASSIGN; state=OFFLINE, location=64035cbb0b46,40081,1733192624478; forceNewPlan=false, retain=false 2024-12-03T02:23:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:48,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=badaa07e05f1c98a805a3234eb1add78, regionState=OPENING, regionLocation=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:48,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=badaa07e05f1c98a805a3234eb1add78, ASSIGN because future has completed 2024-12-03T02:23:48,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure badaa07e05f1c98a805a3234eb1add78, server=64035cbb0b46,40081,1733192624478}] 2024-12-03T02:23:49,145 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:49,145 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T02:23:49,146 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep 2024-12-03T02:23:49,150 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=64035cbb0b46%2C40081%2C1733192624478.rep, suffix=, logDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478, archiveDir=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs, maxLogs=32 2024-12-03T02:23:49,167 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.rep.1733192629152, exclude list is [], retry=0 2024-12-03T02:23:49,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33433,DS-29454614-db68-4ec7-803d-98844b639c75,DISK] 2024-12-03T02:23:49,172 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.rep.1733192629152 2024-12-03T02:23:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37577:37577)] 2024-12-03T02:23:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => badaa07e05f1c98a805a3234eb1add78, NAME => 'hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.', STARTKEY => '', ENDKEY => ''} 2024-12-03T02:23:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T02:23:49,173 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. service=MultiRowMutationService 2024-12-03T02:23:49,174 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully. 2024-12-03T02:23:49,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T02:23:49,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,174 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,176 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,179 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region badaa07e05f1c98a805a3234eb1add78 columnFamilyName hfileref 2024-12-03T02:23:49,179 DEBUG [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:49,180 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(327): Store=badaa07e05f1c98a805a3234eb1add78/hfileref, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:49,180 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,182 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region badaa07e05f1c98a805a3234eb1add78 columnFamilyName queue 2024-12-03T02:23:49,182 DEBUG [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:49,183 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(327): Store=badaa07e05f1c98a805a3234eb1add78/queue, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:49,183 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,185 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region badaa07e05f1c98a805a3234eb1add78 columnFamilyName sid 2024-12-03T02:23:49,185 DEBUG [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T02:23:49,186 INFO [StoreOpener-badaa07e05f1c98a805a3234eb1add78-1 {}] regionserver.HStore(327): Store=badaa07e05f1c98a805a3234eb1add78/sid, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T02:23:49,186 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,189 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,189 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T02:23:49,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T02:23:49,196 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened badaa07e05f1c98a805a3234eb1add78; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66735708, jitterRate=-0.005560457706451416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T02:23:49,196 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:23:49,197 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for badaa07e05f1c98a805a3234eb1add78: Running coprocessor pre-open hook at 1733192629174Writing region info on filesystem at 1733192629174Initializing all the Stores at 1733192629176 (+2 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192629176Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192629176Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733192629176Cleaning up temporary data from old regions at 1733192629189 (+13 ms)Running coprocessor post-open hooks at 1733192629196 (+7 ms)Region opened successfully at 1733192629197 (+1 ms) 2024-12-03T02:23:49,198 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78., pid=7, masterSystemTime=1733192629138 2024-12-03T02:23:49,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:49,201 INFO [RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:23:49,203 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=badaa07e05f1c98a805a3234eb1add78, regionState=OPEN, openSeqNum=2, regionLocation=64035cbb0b46,40081,1733192624478 2024-12-03T02:23:49,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure badaa07e05f1c98a805a3234eb1add78, server=64035cbb0b46,40081,1733192624478 because future has completed 2024-12-03T02:23:49,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6 2024-12-03T02:23:49,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure badaa07e05f1c98a805a3234eb1add78, server=64035cbb0b46,40081,1733192624478 in 225 msec 2024-12-03T02:23:49,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T02:23:49,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=badaa07e05f1c98a805a3234eb1add78, ASSIGN in 391 msec 2024-12-03T02:23:49,219 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T02:23:49,219 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733192629219"}]},"ts":"1733192629219"} 2024-12-03T02:23:49,224 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta 2024-12-03T02:23:49,226 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T02:23:49,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 865 msec 2024-12-03T02:23:49,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78., hostname=64035cbb0b46,40081,1733192624478, seqNum=2] 2024-12-03T02:23:49,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-03T02:23:49,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:49,317 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:23:49,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:49,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8 2024-12-03T02:23:49,534 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:23:49,589 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:49,591 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T02:23:49,591 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=64035cbb0b46,40081,1733192624478, seqNum=-1] 2024-12-03T02:23:49,592 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T02:23:49,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59157, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-03T02:23:49,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-64035cbb0b46,40081,1733192624478', locateType=CURRENT is [region=hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78., hostname=64035cbb0b46,40081,1733192624478, seqNum=2] 2024-12-03T02:23:49,606 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-03T02:23:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-03T02:23:49,613 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:49,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-12-03T02:23:49,619 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:49,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 243 msec 2024-12-03T02:23:49,622 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:49,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 1.3320 sec 2024-12-03T02:23:49,625 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07d64ba2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27a4d202 2024-12-03T02:23:49,635 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:49,635 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T02:23:50,452 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:50,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:23:50,467 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,468 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4858130a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7158154d 2024-12-03T02:23:50,476 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4858130a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4858130a to 127.0.0.1:64747 2024-12-03T02:23:50,476 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:50,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:50,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:23:50,636 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,638 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54868d0d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55b51c87 2024-12-03T02:23:50,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-03T02:23:50,648 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:50,650 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:50,650 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-03T02:23:50,651 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:50,651 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:50,651 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:50,651 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,653 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b8dba84 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1dd28818 2024-12-03T02:23:50,653 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:50,672 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:50,673 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:50,673 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-03T02:23:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-03T02:23:50,678 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:50,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-03T02:23:50,683 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:50,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-03T02:23:50,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 226 msec 2024-12-03T02:23:50,690 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,691 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a34226a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ad19b79 2024-12-03T02:23:50,701 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:50,701 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-03T02:23:50,781 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:50,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:50,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:50,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:50,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:23:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:23:50,799 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,800 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61811c0a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76419ae3 2024-12-03T02:23:50,809 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x61811c0a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61811c0a to 127.0.0.1:64747 2024-12-03T02:23:50,809 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:50,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:23:50,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-03T02:23:50,981 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:50,983 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:50,984 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:50,984 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:50,984 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:50,988 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77a7e1cf to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a40cd1e 2024-12-03T02:23:50,988 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:51,012 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:51,012 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:51,012 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-03T02:23:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-03T02:23:51,020 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:51,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T02:23:51,025 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:23:51,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-03T02:23:51,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 231 msec 2024-12-03T02:23:51,031 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,034 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dea03ea to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fe73719 2024-12-03T02:23:51,043 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:51,043 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-03T02:23:51,121 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:51,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-03T02:23:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:51,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:23:51,134 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,135 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x423caae9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b295e09 2024-12-03T02:23:51,142 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x423caae9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x423caae9 to 127.0.0.1:64747 2024-12-03T02:23:51,142 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:51,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:23:51,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-03T02:23:51,334 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:51,336 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:51,336 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:51,336 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:51,336 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,338 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e0e72e9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20d70217 2024-12-03T02:23:51,339 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:51,361 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:51,361 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:51,361 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-03T02:23:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-03T02:23:51,367 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:51,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T02:23:51,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-03T02:23:51,372 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-03T02:23:51,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 242 msec 2024-12-03T02:23:51,379 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,381 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x679e5a0c to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c8efebb 2024-12-03T02:23:51,393 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:51,393 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T02:23:51,451 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:51,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:23:51,460 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,461 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54444547 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30e8c7b4 2024-12-03T02:23:51,468 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x54444547 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54444547 to 127.0.0.1:64747 2024-12-03T02:23:51,468 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:51,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:23:51,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-03T02:23:51,658 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:51,662 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:51,662 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:51,662 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:51,662 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,667 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21beaf3b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2da5307a 2024-12-03T02:23:51,668 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:51,685 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:51,685 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:51,685 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-03T02:23:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-03T02:23:51,689 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:51,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-03T02:23:51,693 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:51,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-03T02:23:51,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 236 msec 2024-12-03T02:23:51,700 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,702 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77262514 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e759929 2024-12-03T02:23:51,709 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:51,709 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-03T02:23:51,783 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:51,785 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:51,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-03T02:23:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:23:51,793 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,795 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c203047 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f3aaf60 2024-12-03T02:23:51,801 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3c203047 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c203047 to 127.0.0.1:64747 2024-12-03T02:23:51,801 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:51,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:23:51,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-03T02:23:51,974 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:51,977 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:51,977 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:51,977 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:51,977 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:51,980 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6df436bd to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10bba33b 2024-12-03T02:23:51,981 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:52,003 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:52,003 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:52,003 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-03T02:23:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-03T02:23:52,008 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:52,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T02:23:52,012 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-03T02:23:52,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-03T02:23:52,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 223 msec 2024-12-03T02:23:52,020 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,021 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1009b800 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60df218b 2024-12-03T02:23:52,058 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:52,059 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-03T02:23:52,112 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:52,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:52,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:23:52,122 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,124 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74ff7489 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e563bd8 2024-12-03T02:23:52,134 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x74ff7489 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74ff7489 to 127.0.0.1:64747 2024-12-03T02:23:52,134 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:52,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:23:52,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-03T02:23:52,309 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:52,312 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:52,312 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:52,313 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:52,313 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,317 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a0eaccc to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43daa7e4 2024-12-03T02:23:52,317 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:52,334 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:52,334 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:52,334 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-03T02:23:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-03T02:23:52,338 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:52,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T02:23:52,342 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:52,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-03T02:23:52,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 227 msec 2024-12-03T02:23:52,348 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,350 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x189a04a1 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3583e13b 2024-12-03T02:23:52,359 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:52,359 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T02:23:52,441 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:52,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:52,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:52,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T02:23:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:23:52,456 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:52,457 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:52,457 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:52,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:52,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:52,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:52,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:52,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:52,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:52,487 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication' 2024-12-03T02:23:52,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T02:23:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T02:23:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:23:52,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23 2024-12-03T02:23:52,623 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:23:52,624 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:23:52,624 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:23:52,624 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:52,624 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,628 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ca5a3d5 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@201fb1eb 2024-12-03T02:23:52,628 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:52,629 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:23:52,635 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-12-03T02:23:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=23 2024-12-03T02:23:52,640 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:52,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T02:23:52,645 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:23:52,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 172 msec 2024-12-03T02:23:52,646 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:23:52,655 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:23:52,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 208 msec 2024-12-03T02:23:52,659 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:52,666 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=21, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:52,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=21, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=220 msec 2024-12-03T02:23:52,668 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T02:23:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-03T02:23:52,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:52,772 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:52,773 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:52,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-03T02:23:52,777 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:52,778 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:52,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:23:52,781 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-03T02:23:52,883 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:52,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78., hostname=64035cbb0b46,40081,1733192624478, seqNum=2] 2024-12-03T02:23:52,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=269 (was 214) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x189a04a1-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1009b800 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b8dba84-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a0eaccc-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731-prefix:64035cbb0b46,40081,1733192624478.rep java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6df436bd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6e0e72e9-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6e0e72e9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77262514-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x54868d0d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1009b800-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x21beaf3b-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77a7e1cf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3dea03ea-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a34226a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-189877540_22 at /127.0.0.1:54132 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07d64ba2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x679e5a0c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77a7e1cf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0ca5a3d5-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-189877540_22 at /127.0.0.1:54236 [Receiving block BP-1640616088-172.17.0.3-1733192619772:blk_1073741837_1013] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a0eaccc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x54868d0d-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0ca5a3d5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a0eaccc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6df436bd java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x679e5a0c-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x21beaf3b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x189a04a1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x679e5a0c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1009b800-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6e0e72e9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b8dba84-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77a7e1cf-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6df436bd-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77262514-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x54868d0d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0ca5a3d5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a34226a-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07d64ba2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x77262514 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x189a04a1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3dea03ea java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07d64ba2-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: PacketResponder: BP-1640616088-172.17.0.3-1733192619772:blk_1073741837_1013, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a34226a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b8dba84 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3dea03ea-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x21beaf3b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=513 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=531 (was 551), ProcessCount=11 (was 11), AvailableMemoryMB=2368 (was 2616) 2024-12-03T02:23:52,940 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=269, OpenFileDescriptor=513, MaxFileDescriptor=1048576, SystemLoadAverage=531, ProcessCount=11, AvailableMemoryMB=2367 2024-12-03T02:23:52,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:23:52,947 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:52,948 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d55c749 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a802dc5 2024-12-03T02:23:53,022 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0d55c749 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d55c749 to 127.0.0.1:64747 2024-12-03T02:23:53,022 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:53,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:23:53,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:53,226 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:23:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:23:53,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26 2024-12-03T02:23:53,284 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:23:53,313 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:53,316 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-03T02:23:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-03T02:23:53,319 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:53,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T02:23:53,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-03T02:23:53,322 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:53,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 380 msec 2024-12-03T02:23:53,330 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:53,331 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b0cad62 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cbee443 2024-12-03T02:23:53,368 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:53,368 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T02:23:53,571 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:53,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:53,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T02:23:53,575 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:53,576 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4918b1fb to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b549388 2024-12-03T02:23:53,592 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4918b1fb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4918b1fb to 127.0.0.1:64747 2024-12-03T02:23:53,593 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:53,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T02:23:53,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-03T02:23:53,765 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:53,768 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:53,768 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:53,768 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:53,768 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:53,769 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x563e9ce1 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38cd127b 2024-12-03T02:23:53,770 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:53,784 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:53,784 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:53,784 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-03T02:23:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-03T02:23:53,788 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:53,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-03T02:23:53,791 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:53,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec 2024-12-03T02:23:53,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 219 msec 2024-12-03T02:23:53,799 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:53,801 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x097537ee to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ad29eba 2024-12-03T02:23:53,876 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:53,876 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-03T02:23:53,890 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:53,892 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:23:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T02:23:53,896 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:53,899 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78946d08 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@71ac71d 2024-12-03T02:23:53,934 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x78946d08 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x78946d08 to 127.0.0.1:64747 2024-12-03T02:23:53,934 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T02:23:54,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:54,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T02:23:54,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T02:23:54,071 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication 2024-12-03T02:23:54,071 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer 2024-12-03T02:23:54,073 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:23:54,073 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T02:23:54,073 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T02:23:54,073 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T02:23:54,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-03T02:23:54,156 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:54,159 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:54,160 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:54,160 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:54,160 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,163 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x33263965 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14d1c714 2024-12-03T02:23:54,163 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:54,178 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:54,178 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:54,179 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-03T02:23:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-03T02:23:54,183 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:54,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T02:23:54,187 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:23:54,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-03T02:23:54,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 295 msec 2024-12-03T02:23:54,200 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,202 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c5822d9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25debd04 2024-12-03T02:23:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-03T02:23:54,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:54,211 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-03T02:23:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:54,243 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:54,244 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T02:23:54,246 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,249 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x709152ba to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@212db64a 2024-12-03T02:23:54,312 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x709152ba {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x709152ba to 127.0.0.1:64747 2024-12-03T02:23:54,312 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:54,314 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:54,338 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:54,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=127 msec 2024-12-03T02:23:54,339 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T02:23:54,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-03T02:23:54,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:23:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-03T02:23:54,355 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,357 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c4bb589 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@794d777 2024-12-03T02:23:54,409 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x5c4bb589 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c4bb589 to 127.0.0.1:64747 2024-12-03T02:23:54,409 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:54,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-03T02:23:54,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33 2024-12-03T02:23:54,581 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:54,585 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:54,585 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:54,586 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:54,586 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,589 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69de8b29 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@af5a700 2024-12-03T02:23:54,590 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:54,607 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:54,608 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:54,608 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-03T02:23:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-03T02:23:54,612 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:54,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T02:23:54,615 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:23:54,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-03T02:23:54,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 264 msec 2024-12-03T02:23:54,622 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,624 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11887047 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12bc6543 2024-12-03T02:23:54,634 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:54,634 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-03T02:23:54,672 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:54,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:23:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:23:54,677 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,679 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b3368a6 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5acd3128 2024-12-03T02:23:54,692 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x6b3368a6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b3368a6 to 127.0.0.1:64747 2024-12-03T02:23:54,692 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:54,692 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:54,695 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:54,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=22 msec 2024-12-03T02:23:54,697 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-03T02:23:54,783 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-03T02:23:54,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T02:23:54,794 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:54,795 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:54,796 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:54,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:54,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:54,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:54,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:54,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:54,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T02:23:54,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36 2024-12-03T02:23:54,964 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:23:54,965 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:23:54,965 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:23:54,965 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:54,966 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:54,969 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02c7bd30 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4736f882 2024-12-03T02:23:54,970 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:54,971 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:23:54,975 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36 2024-12-03T02:23:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=36 2024-12-03T02:23:54,978 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:54,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-03T02:23:54,981 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:23:54,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 168 msec 2024-12-03T02:23:54,981 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:23:54,986 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:23:54,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 201 msec 2024-12-03T02:23:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-03T02:23:55,110 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:55,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-03T02:23:55,115 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:55,116 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:55,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:23:55,118 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-03T02:23:55,222 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:55,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:23:55,227 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:55,230 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:55,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-03T02:23:55,232 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-03T02:23:55,331 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:55,347 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=293 (was 269) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3c5822d9-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1b0cad62-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x33263965-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x02c7bd30-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x02c7bd30-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x563e9ce1-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x33263965-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x02c7bd30 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3c5822d9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x33263965 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1b0cad62-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69de8b29-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x097537ee-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x11887047-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x097537ee java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69de8b29 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1b0cad62 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x097537ee-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x11887047 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x563e9ce1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x563e9ce1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69de8b29-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x11887047-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3c5822d9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=543 (was 513) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=531 (was 531), ProcessCount=11 (was 11), AvailableMemoryMB=2252 (was 2367) 2024-12-03T02:23:55,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=293, OpenFileDescriptor=543, MaxFileDescriptor=1048576, SystemLoadAverage=531, ProcessCount=11, AvailableMemoryMB=2251 2024-12-03T02:23:55,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-03T02:23:55,361 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:55,362 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17637455 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@64cbcb84 2024-12-03T02:23:55,375 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x17637455 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17637455 to 127.0.0.1:64747 2024-12-03T02:23:55,375 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:55,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-03T02:23:55,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40 2024-12-03T02:23:55,554 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:23:55,586 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:55,589 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-03T02:23:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-03T02:23:55,592 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:55,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-03T02:23:55,595 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:55,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-03T02:23:55,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 237 msec 2024-12-03T02:23:55,602 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:55,603 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13563e75 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8c0bf13 2024-12-03T02:23:55,609 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:55,609 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-03T02:23:55,682 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:55,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-03T02:23:55,690 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:55,691 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75c9dc9f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@65259579 2024-12-03T02:23:55,700 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x75c9dc9f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75c9dc9f to 127.0.0.1:64747 2024-12-03T02:23:55,701 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:55,701 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:55,702 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:55,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=19 msec 2024-12-03T02:23:55,704 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-03T02:23:55,803 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-03T02:23:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:23:55,814 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:23:55,817 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:55,818 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:55,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:23:55,820 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-03T02:23:55,921 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:23:55,923 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-03T02:23:55,927 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:23:55,928 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3492ba7e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46f2b835 2024-12-03T02:23:55,934 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3492ba7e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3492ba7e to 127.0.0.1:64747 2024-12-03T02:23:55,934 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:56,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-03T02:23:56,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44 2024-12-03T02:23:56,159 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-03T02:23:56,186 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-64035cbb0b46,40081,1733192624478, ReplicationSource: 2, currentBandwidth=0 2024-12-03T02:23:56,190 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-12-03T02:23:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=44 2024-12-03T02:23:56,195 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:56,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-03T02:23:56,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-03T02:23:56,199 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:56,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 275 msec 2024-12-03T02:23:56,209 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:23:56,210 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49523672 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52b52b3 2024-12-03T02:23:56,226 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:56,226 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-03T02:23:56,241 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-03T02:23:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:23:56,243 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-03T02:23:56,249 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:56,250 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:56,251 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:56,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:56,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:56,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:56,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:56,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:56,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-03T02:23:56,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-03T02:23:56,415 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:23:56,416 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:23:56,416 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:23:56,416 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:56,416 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:56,418 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a4b8f39 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f983c71 2024-12-03T02:23:56,419 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:56,420 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:23:56,423 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-03T02:23:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=46 2024-12-03T02:23:56,430 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:56,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-12-03T02:23:56,434 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:23:56,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 170 msec 2024-12-03T02:23:56,434 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:23:56,447 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:23:56,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 204 msec 2024-12-03T02:23:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-03T02:23:56,561 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:23:56,564 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-03T02:23:56,571 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-03T02:23:56,572 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-03T02:23:56,572 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-03T02:23:56,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-03T02:23:56,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-03T02:23:56,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-03T02:23:56,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-03T02:23:56,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-03T02:23:56,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-03T02:23:56,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-03T02:23:56,739 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-03T02:23:56,739 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-03T02:23:56,740 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:23:56,740 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-03T02:23:56,740 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:23:56,741 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f58486b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25865dd4 2024-12-03T02:23:56,742 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:56,742 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-64035cbb0b46,40081,1733192624478 2024-12-03T02:23:56,745 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-03T02:23:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-03T02:23:56,748 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:56,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-03T02:23:56,751 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-03T02:23:56,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-03T02:23:56,751 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-03T02:23:56,760 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-03T02:23:56,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 196 msec 2024-12-03T02:23:56,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-03T02:23:56,891 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-03T02:23:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:23:56,893 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-03T02:23:56,896 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:56,898 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:56,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:23:56,899 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-03T02:23:57,002 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:57,002 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:23:57,006 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:57,008 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:23:57,010 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-03T02:23:57,110 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:57,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=305 (was 293) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f58486b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4b8f39 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f58486b-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x49523672-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4b8f39-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x49523672-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13563e75 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x49523672 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4b8f39-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13563e75-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f58486b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13563e75-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=559 (was 543) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 531), ProcessCount=11 (was 11), AvailableMemoryMB=2127 (was 2251) 2024-12-03T02:23:57,136 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=305, OpenFileDescriptor=559, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=2126 2024-12-03T02:23:57,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-03T02:23:57,141 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-03T02:23:57,143 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-12-03T02:23:57,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-03T02:23:57,253 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-03T02:23:57,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T02:23:57,263 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:57,264 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:23:57,266 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-03T02:23:57,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:57,372 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T02:23:57,374 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:57,376 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:23:57,377 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-03T02:23:57,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:57,499 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=305 (was 305), OpenFileDescriptor=559 (was 559), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=2125 (was 2126) 2024-12-03T02:23:57,509 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=305, OpenFileDescriptor=559, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=2125 2024-12-03T02:23:57,510 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T02:23:57,514 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-03T02:23:57,515 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-12-03T02:23:57,517 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-03T02:23:57,621 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-03T02:23:57,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:23:57,626 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:57,628 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-03T02:23:57,631 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-03T02:23:57,732 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:57,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:57,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T02:23:57,737 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:57,740 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:57,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-03T02:23:57,742 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-03T02:23:57,841 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:57,856 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=304 (was 305), OpenFileDescriptor=559 (was 559), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=2119 (was 2125) 2024-12-03T02:23:57,865 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=304, OpenFileDescriptor=559, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=2118 2024-12-03T02:23:57,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T02:23:57,869 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:57,870 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x577274be to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74f9f6a3 2024-12-03T02:23:57,875 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x577274be {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x577274be to 127.0.0.1:64747 2024-12-03T02:23:57,876 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:57,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T02:23:58,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58 2024-12-03T02:23:58,066 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:23:58,111 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:58,114 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-12-03T02:23:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-12-03T02:23:58,117 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:58,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-03T02:23:58,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 207 msec 2024-12-03T02:23:58,120 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:58,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 254 msec 2024-12-03T02:23:58,129 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,130 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fde21a0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@66db1f4c 2024-12-03T02:23:58,132 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:23:58,142 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:58,142 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-03T02:23:58,191 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:58,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:58,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T02:23:58,197 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,198 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x473d48e5 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24d8c05d 2024-12-03T02:23:58,209 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x473d48e5 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x473d48e5 to 127.0.0.1:64747 2024-12-03T02:23:58,209 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:58,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T02:23:58,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-03T02:23:58,380 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:58,382 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:58,382 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:58,382 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:58,382 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,384 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62d0ea85 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2dda112e 2024-12-03T02:23:58,385 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:58,404 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:58,404 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:58,404 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-03T02:23:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-12-03T02:23:58,408 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:58,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-03T02:23:58,410 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:23:58,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-03T02:23:58,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 218 msec 2024-12-03T02:23:58,417 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,418 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c3b08b3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3045114c 2024-12-03T02:23:58,425 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:58,425 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-03T02:23:58,511 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:58,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-03T02:23:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-03T02:23:58,515 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,516 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x314538ea to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@fe00a40 2024-12-03T02:23:58,526 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x314538ea {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x314538ea to 127.0.0.1:64747 2024-12-03T02:23:58,526 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:58,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-03T02:23:58,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-03T02:23:58,701 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:58,705 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:58,705 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:58,705 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:58,705 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,708 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c4b7649 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f88ffc7 2024-12-03T02:23:58,708 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:58,727 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:58,728 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:58,728 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-03T02:23:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-03T02:23:58,732 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:58,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-03T02:23:58,735 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-03T02:23:58,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-03T02:23:58,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 224 msec 2024-12-03T02:23:58,743 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,745 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x326ca2dc to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72d97312 2024-12-03T02:23:58,750 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:58,750 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-03T02:23:58,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:58,832 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:58,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:23:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:23:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T02:23:58,838 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:58,839 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x764bfc83 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ceaf3df 2024-12-03T02:23:58,851 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x764bfc83 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x764bfc83 to 127.0.0.1:64747 2024-12-03T02:23:58,851 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:58,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:58,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T02:23:59,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-03T02:23:59,021 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:23:59,024 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:23:59,024 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:23:59,025 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:59,025 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:59,027 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5dcda8da to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5051b96 2024-12-03T02:23:59,028 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:59,045 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:59,045 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:23:59,045 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-03T02:23:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=64 2024-12-03T02:23:59,049 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:59,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-03T02:23:59,052 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:23:59,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-03T02:23:59,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 219 msec 2024-12-03T02:23:59,060 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:59,061 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b15f4c5 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@678f8aa1 2024-12-03T02:23:59,067 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:23:59,067 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:23:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-03T02:23:59,150 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:23:59,151 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:23:59,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-03T02:23:59,158 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:59,159 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:59,160 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:23:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:23:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:23:59,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:23:59,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-03T02:23:59,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-03T02:23:59,329 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:23:59,330 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:23:59,330 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:23:59,330 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:23:59,330 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:59,332 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4beb3e50 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3796c215 2024-12-03T02:23:59,333 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:23:59,333 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:23:59,337 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-03T02:23:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-03T02:23:59,340 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:59,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-03T02:23:59,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-03T02:23:59,343 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:23:59,343 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:23:59,350 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:23:59,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 198 msec 2024-12-03T02:23:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-03T02:23:59,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:23:59,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:23:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T02:23:59,474 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:59,475 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:59,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:23:59,477 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-03T02:23:59,581 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:23:59,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:23:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:23:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:23:59,584 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:23:59,585 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:23:59,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:23:59,587 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:23:59,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-03T02:23:59,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:23:59,715 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=325 (was 304) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7fde21a0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7fde21a0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c3b08b3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2b15f4c5-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c4b7649-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4beb3e50-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x326ca2dc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c4b7649-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5dcda8da java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4beb3e50-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x62d0ea85-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5dcda8da-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x62d0ea85-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c3b08b3-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2b15f4c5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4beb3e50 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x326ca2dc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c3b08b3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6c4b7649 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x326ca2dc-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7fde21a0-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2b15f4c5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5dcda8da-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x62d0ea85 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=587 (was 559) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=2066 (was 2118) 2024-12-03T02:23:59,726 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=325, OpenFileDescriptor=587, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=2065 2024-12-03T02:23:59,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:23:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:23:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-03T02:23:59,732 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:23:59,734 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a49e1c1 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a372815 2024-12-03T02:23:59,759 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3a49e1c1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a49e1c1 to 127.0.0.1:64747 2024-12-03T02:23:59,760 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:23:59,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:23:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-03T02:23:59,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70 2024-12-03T02:23:59,940 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:23:59,985 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:23:59,989 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70 2024-12-03T02:23:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=70 2024-12-03T02:23:59,992 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:23:59,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-03T02:23:59,995 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:23:59,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 207 msec 2024-12-03T02:23:59,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 267 msec 2024-12-03T02:24:00,004 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,005 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6289e6c3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34fa6e02 2024-12-03T02:24:00,039 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:00,039 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-03T02:24:00,050 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:00,050 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-03T02:24:00,053 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,054 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d48a1f6 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@638dbd 2024-12-03T02:24:00,075 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4d48a1f6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d48a1f6 to 127.0.0.1:64747 2024-12-03T02:24:00,075 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:00,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-03T02:24:00,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-03T02:24:00,247 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:00,251 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:00,251 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:00,252 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:00,252 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,254 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x542540fb to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@38219851 2024-12-03T02:24:00,254 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:00,271 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:00,271 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:00,272 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-03T02:24:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=72 2024-12-03T02:24:00,274 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:00,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-12-03T02:24:00,277 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:00,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-03T02:24:00,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 227 msec 2024-12-03T02:24:00,287 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,288 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f9f452f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@653e8584 2024-12-03T02:24:00,300 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:00,301 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-03T02:24:00,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:00,373 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:00,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-03T02:24:00,381 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,383 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x254f084e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bf48d83 2024-12-03T02:24:00,392 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x254f084e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x254f084e to 127.0.0.1:64747 2024-12-03T02:24:00,393 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:00,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-03T02:24:00,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-03T02:24:00,562 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:00,563 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:00,563 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:00,563 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:00,563 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,565 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6bec7467 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60545a43 2024-12-03T02:24:00,565 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:00,586 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:00,586 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:00,587 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-03T02:24:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=74 2024-12-03T02:24:00,589 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:00,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-03T02:24:00,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-03T02:24:00,591 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:00,593 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 216 msec 2024-12-03T02:24:00,600 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,601 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x220dcd5d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@eabb564 2024-12-03T02:24:00,609 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:00,609 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-03T02:24:00,701 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:00,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:00,703 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:00,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-03T02:24:00,709 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,711 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26f99f6e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@996be87 2024-12-03T02:24:00,717 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x26f99f6e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x26f99f6e to 127.0.0.1:64747 2024-12-03T02:24:00,717 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:00,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-03T02:24:00,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-03T02:24:00,887 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:00,889 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:00,889 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:00,889 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:00,890 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,892 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ff40707 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c76066b 2024-12-03T02:24:00,893 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:00,910 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:00,910 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:00,910 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-03T02:24:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=76 2024-12-03T02:24:00,914 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:00,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-03T02:24:00,916 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:00,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-03T02:24:00,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 211 msec 2024-12-03T02:24:00,926 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:00,928 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x431bd9f0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c58a37c 2024-12-03T02:24:00,933 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:00,934 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-03T02:24:01,033 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:01,034 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,040 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:01,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:24:01,043 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,045 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x413ccb01 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a0786ee 2024-12-03T02:24:01,050 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x413ccb01 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x413ccb01 to 127.0.0.1:64747 2024-12-03T02:24:01,050 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:01,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:24:01,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-03T02:24:01,222 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:01,223 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:01,223 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:01,223 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:01,223 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,225 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1787e7ee to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e47d2be 2024-12-03T02:24:01,225 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:01,242 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:01,242 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:01,243 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-03T02:24:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=78 2024-12-03T02:24:01,245 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:01,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-03T02:24:01,247 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:01,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec 2024-12-03T02:24:01,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 207 msec 2024-12-03T02:24:01,256 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,257 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74505115 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@105a3588 2024-12-03T02:24:01,267 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:01,267 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-03T02:24:01,361 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:01,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T02:24:01,368 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,369 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x194c1b0d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2df50f15 2024-12-03T02:24:01,375 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x194c1b0d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x194c1b0d to 127.0.0.1:64747 2024-12-03T02:24:01,375 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:01,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T02:24:01,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-03T02:24:01,546 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:01,547 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:01,547 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:01,547 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:01,547 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,548 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7412793c to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39ab79a 2024-12-03T02:24:01,549 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:01,563 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:01,564 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:01,564 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-03T02:24:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=80 2024-12-03T02:24:01,567 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:01,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-03T02:24:01,571 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:01,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-03T02:24:01,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 207 msec 2024-12-03T02:24:01,579 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,580 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a95f20b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55cef8e3 2024-12-03T02:24:01,592 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:01,592 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-03T02:24:01,681 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:01,682 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,683 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:01,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T02:24:01,688 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,688 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2983b5ce to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ae59878 2024-12-03T02:24:01,700 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x2983b5ce {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2983b5ce to 127.0.0.1:64747 2024-12-03T02:24:01,700 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:01,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T02:24:01,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-03T02:24:01,872 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:01,874 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:01,874 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:01,875 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:01,875 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,876 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3926b201 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b9c1d9d 2024-12-03T02:24:01,877 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:01,899 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:01,899 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:01,899 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-03T02:24:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-12-03T02:24:01,902 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:01,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-03T02:24:01,904 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:01,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-03T02:24:01,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec 2024-12-03T02:24:01,913 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:01,914 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6629d996 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bb0dddf 2024-12-03T02:24:01,925 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:01,926 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-03T02:24:02,002 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:02,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:02,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-03T02:24:02,009 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,010 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4006dff0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21f8770 2024-12-03T02:24:02,017 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4006dff0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4006dff0 to 127.0.0.1:64747 2024-12-03T02:24:02,017 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:02,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-03T02:24:02,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-03T02:24:02,189 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:02,192 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-03T02:24:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-12-03T02:24:02,196 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:02,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-03T02:24:02,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-03T02:24:02,201 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:02,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 196 msec 2024-12-03T02:24:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-03T02:24:02,332 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:02,333 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:02,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:02,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-03T02:24:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:24:02,341 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,342 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4c557c29 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9984d55 2024-12-03T02:24:02,419 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4c557c29 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4c557c29 to 127.0.0.1:64747 2024-12-03T02:24:02,419 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:24:02,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:02,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-03T02:24:02,615 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:02,618 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:02,619 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:02,619 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:02,619 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,622 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78c4d062 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74ee98f8 2024-12-03T02:24:02,622 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:02,636 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:02,637 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:02,637 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-03T02:24:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-03T02:24:02,640 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:02,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-03T02:24:02,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-03T02:24:02,642 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-03T02:24:02,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 305 msec 2024-12-03T02:24:02,651 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,652 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6fc6183d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ca37078 2024-12-03T02:24:02,658 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:02,658 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-03T02:24:02,660 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:02,660 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:02,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-03T02:24:02,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:02,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:24:02,664 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,665 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75ca98fd to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7dd0ea26 2024-12-03T02:24:02,675 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x75ca98fd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75ca98fd to 127.0.0.1:64747 2024-12-03T02:24:02,675 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:02,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:24:02,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-03T02:24:02,845 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:02,846 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:02,846 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:02,846 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:02,847 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,848 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fc0661a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f915e2d 2024-12-03T02:24:02,848 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:02,864 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:02,864 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:02,864 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-03T02:24:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-03T02:24:02,867 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:02,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-03T02:24:02,869 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-03T02:24:02,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-03T02:24:02,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 208 msec 2024-12-03T02:24:02,882 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:02,883 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75d82551 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@189ff3b3 2024-12-03T02:24:02,892 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:02,892 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-03T02:24:02,980 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:02,980 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:02,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:24:02,985 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:02,986 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:02,986 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:02,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:03,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:03,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:03,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:03,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:03,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:24:03,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-03T02:24:03,155 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:03,155 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:03,155 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:03,155 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:03,155 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:03,157 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x506ce346 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3daea3a5 2024-12-03T02:24:03,157 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:03,158 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:03,160 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-03T02:24:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-03T02:24:03,163 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:03,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-03T02:24:03,165 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:03,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-03T02:24:03,165 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:03,170 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:03,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 189 msec 2024-12-03T02:24:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-03T02:24:03,301 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:03,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:24:03,304 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:03,306 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:03,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:03,308 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-03T02:24:03,411 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:03,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T02:24:03,415 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:03,417 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:03,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:03,419 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-03T02:24:03,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:03,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=379 (was 325) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1ff40707-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1787e7ee-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6289e6c3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x220dcd5d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4f9f452f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1ff40707-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3926b201-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x75d82551 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6289e6c3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7412793c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6fc6183d-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7a95f20b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6629d996-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3926b201 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1787e7ee-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x542540fb-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x220dcd5d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6fc6183d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6bec7467 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6629d996-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x74505115 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x220dcd5d-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1787e7ee java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x431bd9f0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7412793c-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7412793c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1fc0661a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1ff40707 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4f9f452f-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x74505115-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6fc6183d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x542540fb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6bec7467-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x506ce346 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1fc0661a-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x506ce346-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1fc0661a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x78c4d062-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x506ce346-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x431bd9f0-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x431bd9f0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6289e6c3-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x78c4d062-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6bec7467-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4f9f452f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x75d82551-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3926b201-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x78c4d062 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7a95f20b-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7a95f20b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x74505115-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x75d82551-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6629d996 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x542540fb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=657 (was 587) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 521), ProcessCount=11 (was 11), AvailableMemoryMB=1936 (was 2065) 2024-12-03T02:24:03,559 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=379, OpenFileDescriptor=657, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=1934 2024-12-03T02:24:03,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:24:03,564 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:03,565 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4fc36dbd to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73ac977f 2024-12-03T02:24:03,575 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x4fc36dbd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4fc36dbd to 127.0.0.1:64747 2024-12-03T02:24:03,575 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:03,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:24:03,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94 2024-12-03T02:24:03,789 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:03,832 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:03,836 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-03T02:24:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-03T02:24:03,839 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:03,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-03T02:24:03,844 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:03,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 204 msec 2024-12-03T02:24:03,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 284 msec 2024-12-03T02:24:03,857 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:03,859 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d9365b6 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52aaa11e 2024-12-03T02:24:03,875 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:03,875 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-03T02:24:03,881 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:03,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-03T02:24:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-03T02:24:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:24:03,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:24:04,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-03T02:24:04,054 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-03T02:24:04,055 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:04,055 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:04,055 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:04,055 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:04,057 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x027dc39c to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e258b3a 2024-12-03T02:24:04,057 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:04,075 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:04,075 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:04,076 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-03T02:24:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-03T02:24:04,080 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:04,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-03T02:24:04,084 INFO [PEWorker-5 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-03T02:24:04,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-03T02:24:04,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 197 msec 2024-12-03T02:24:04,096 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:04,098 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f5e4f5f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@304aead 2024-12-03T02:24:04,108 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:04,109 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-03T02:24:04,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:04,211 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:24:04,216 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:04,216 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:04,216 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:04,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:04,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:04,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:04,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:04,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:04,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:24:04,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-03T02:24:04,379 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:04,379 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:04,379 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:04,379 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:04,380 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:04,381 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5db84834 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e6ed69f 2024-12-03T02:24:04,381 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:04,381 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:04,383 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-03T02:24:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-03T02:24:04,385 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:04,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-03T02:24:04,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-03T02:24:04,387 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:04,387 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:04,393 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:04,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 181 msec 2024-12-03T02:24:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-03T02:24:04,531 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:04,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:24:04,534 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:04,536 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:04,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:04,538 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-03T02:24:04,642 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:04,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-03T02:24:04,646 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:04,648 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:04,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:24:04,649 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-03T02:24:04,752 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:04,772 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=390 (was 379) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f5e4f5f-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3d9365b6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5db84834-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f5e4f5f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0f5e4f5f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3d9365b6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x027dc39c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5db84834 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5db84834-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x027dc39c-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3d9365b6-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x027dc39c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=673 (was 657) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=1855 (was 1934) 2024-12-03T02:24:04,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=390, OpenFileDescriptor=673, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=1855 2024-12-03T02:24:04,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-03T02:24:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:24:04,787 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:04,788 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:04,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:04,790 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:04,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:04,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-03T02:24:04,891 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-03T02:24:04,892 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-03T02:24:04,895 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:04,896 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:04,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:04,898 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-03T02:24:05,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:05,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:05,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:05,008 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:05,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:24:05,009 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:05,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:05,011 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-03T02:24:05,121 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:05,141 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=391 (was 390) - Thread LEAK? -, OpenFileDescriptor=673 (was 673), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=1846 (was 1855) 2024-12-03T02:24:05,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=391, OpenFileDescriptor=673, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=1845 2024-12-03T02:24:05,152 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-03T02:24:05,156 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:05,157 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51bd8bd0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c90eb72 2024-12-03T02:24:05,184 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x51bd8bd0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51bd8bd0 to 127.0.0.1:64747 2024-12-03T02:24:05,184 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:05,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-03T02:24:05,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105 2024-12-03T02:24:05,363 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:05,398 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:05,400 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-12-03T02:24:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=105 2024-12-03T02:24:05,402 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:05,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-03T02:24:05,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-03T02:24:05,404 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:05,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 251 msec 2024-12-03T02:24:05,415 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:05,416 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63c42631 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d5a1066 2024-12-03T02:24:05,425 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:05,425 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-03T02:24:05,470 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:05,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T02:24:05,479 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:05,480 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:05,481 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:05,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:05,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:05,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:05,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:05,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:05,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T02:24:05,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-03T02:24:05,646 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:05,646 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:05,647 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:05,647 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:05,647 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:05,648 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53688c72 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f9b0242 2024-12-03T02:24:05,648 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:05,649 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:05,651 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-03T02:24:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=107 2024-12-03T02:24:05,653 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:05,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-03T02:24:05,655 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:05,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-03T02:24:05,655 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:05,661 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:05,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 189 msec 2024-12-03T02:24:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-03T02:24:05,791 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:05,791 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-03T02:24:05,795 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:05,797 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:05,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:24:05,798 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-03T02:24:05,901 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:05,901 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T02:24:05,904 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:05,905 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:05,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:05,907 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-03T02:24:06,011 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:06,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=397 (was 391) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x53688c72 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x53688c72-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63c42631-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63c42631-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x53688c72-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63c42631 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=681 (was 673) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 487), ProcessCount=11 (was 11), AvailableMemoryMB=1825 (was 1845) 2024-12-03T02:24:06,041 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=397, OpenFileDescriptor=681, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=1824 2024-12-03T02:24:06,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.CallRunner(138): callId: 308 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:39128 deadline: 1733192706049, exception=java.io.IOException: Replication peer modification disabled 2024-12-03T02:24:06,075 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 24 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:24:06,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.CallRunner(138): callId: 309 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:39128 deadline: 1733192706180, exception=java.io.IOException: Replication peer modification disabled 2024-12-03T02:24:06,181 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 132 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:24:06,183 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-03T02:24:06.076Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:24:06.182Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:24:06,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:06,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T02:24:06,189 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:06,191 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:06,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:06,193 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-03T02:24:06,302 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:06,303 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T02:24:06,309 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:06,311 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:06,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-03T02:24:06,312 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:06,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-03T02:24:06,421 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:06,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=397 (was 397), OpenFileDescriptor=681 (was 681), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 448), ProcessCount=11 (was 11), AvailableMemoryMB=1832 (was 1824) - AvailableMemoryMB LEAK? - 2024-12-03T02:24:06,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=397, OpenFileDescriptor=681, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=1831 2024-12-03T02:24:06,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:06,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:24:06,455 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:06,456 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a282b36 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39d79a3c 2024-12-03T02:24:06,483 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x7a282b36 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a282b36 to 127.0.0.1:64747 2024-12-03T02:24:06,483 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:06,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:06,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:24:06,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113 2024-12-03T02:24:06,692 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:06,727 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:06,729 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-12-03T02:24:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=113 2024-12-03T02:24:06,731 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:06,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-03T02:24:06,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-03T02:24:06,733 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:06,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 282 msec 2024-12-03T02:24:06,743 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:06,744 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5408e30e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f78e444 2024-12-03T02:24:06,767 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:06,767 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:06,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-03T02:24:06,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:06,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:06,772 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:06,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T02:24:06,775 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:06,776 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:06,776 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:06,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:06,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:06,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:06,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:06,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:06,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T02:24:06,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-03T02:24:06,937 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:06,937 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:06,937 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:06,937 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:06,937 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:06,939 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x607c121a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58ac5afa 2024-12-03T02:24:06,940 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:06,941 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:06,945 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-03T02:24:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=115 2024-12-03T02:24:06,948 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:06,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-03T02:24:06,950 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:06,950 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:06,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-03T02:24:06,955 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:06,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 183 msec 2024-12-03T02:24:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-03T02:24:07,090 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:07,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:07,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T02:24:07,093 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:07,095 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:07,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:07,097 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-03T02:24:07,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:07,221 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=403 (was 397) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5408e30e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5408e30e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5408e30e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x607c121a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x607c121a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x607c121a-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=689 (was 681) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 448), ProcessCount=11 (was 11), AvailableMemoryMB=1817 (was 1831) 2024-12-03T02:24:07,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=403, OpenFileDescriptor=689, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=1816 2024-12-03T02:24:07,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:24:07,236 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:07,237 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70d76217 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31eb328e 2024-12-03T02:24:07,249 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x70d76217 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70d76217 to 127.0.0.1:64747 2024-12-03T02:24:07,249 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:07,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:24:07,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118 2024-12-03T02:24:07,430 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:07,463 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:07,465 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-03T02:24:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-03T02:24:07,468 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:07,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-03T02:24:07,470 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:07,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-03T02:24:07,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 237 msec 2024-12-03T02:24:07,478 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:07,479 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07ac0126 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@149f0fec 2024-12-03T02:24:07,500 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:07,500 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:07,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-03T02:24:07,552 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:07,553 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:07,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-03T02:24:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:24:07,561 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:07,562 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1fda47bb to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@752022c3 2024-12-03T02:24:07,574 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x1fda47bb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1fda47bb to 127.0.0.1:64747 2024-12-03T02:24:07,575 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:07,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:24:07,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-03T02:24:07,746 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:07,748 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-03T02:24:07,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-03T02:24:07,752 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:07,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-03T02:24:07,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-03T02:24:07,755 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-03T02:24:07,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 199 msec 2024-12-03T02:24:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-03T02:24:07,882 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:07,883 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:07,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:24:07,889 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:07,890 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:07,890 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:07,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:07,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:07,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:07,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:07,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:07,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:24:08,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-03T02:24:08,070 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:08,070 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:08,070 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:08,070 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:08,070 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:08,072 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x59a08698 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ca60513 2024-12-03T02:24:08,072 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:08,072 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:08,075 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-03T02:24:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-03T02:24:08,078 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:08,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-03T02:24:08,081 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:08,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec 2024-12-03T02:24:08,081 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:08,090 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:08,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 205 msec 2024-12-03T02:24:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-03T02:24:08,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:08,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:24:08,205 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:08,207 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:08,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:08,208 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-03T02:24:08,312 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:08,313 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-03T02:24:08,316 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:08,318 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:08,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-03T02:24:08,320 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-03T02:24:08,421 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:08,448 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=409 (was 403) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07ac0126-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07ac0126 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x07ac0126-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x59a08698-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x59a08698-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x59a08698 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=697 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 448), ProcessCount=11 (was 11), AvailableMemoryMB=1792 (was 1816) 2024-12-03T02:24:08,458 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=409, OpenFileDescriptor=697, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=1791 2024-12-03T02:24:08,460 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:24:08,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:24:08,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126 2024-12-03T02:24:08,638 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:08,670 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:08,673 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-03T02:24:08,673 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-64035cbb0b46,40081,1733192624478 (queues=1) is replicating from cluster=5b5d6b8d-1904-4383-be12-bce3422a64f5 to cluster=5b5d6b8d-1904-4383-be12-bce3422a64f5 2024-12-03T02:24:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-03T02:24:08,674 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=64035cbb0b46%2C40081%2C1733192624478 2024-12-03T02:24:08,675 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:08,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-03T02:24:08,677 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:08,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-03T02:24:08,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 217 msec 2024-12-03T02:24:08,679 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-64035cbb0b46,40081,1733192624478, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-03T02:24:08,681 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 64035cbb0b46%2C40081%2C1733192624478 2024-12-03T02:24:08,685 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.wal-reader.64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.1733192626668, startPosition=0, beingWritten=true 2024-12-03T02:24:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-03T02:24:08,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:08,781 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:24:08,783 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-03T02:24:08,784 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:08,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec 2024-12-03T02:24:08,786 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-03T02:24:08,892 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-03T02:24:08,893 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:08,899 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:08,900 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:08,901 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:08,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:09,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129 2024-12-03T02:24:09,087 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:09,087 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:09,087 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:09,884 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:10,088 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.wal-reader.64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-03T02:24:10,088 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-03T02:24:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:11,088 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 terminated 2024-12-03T02:24:11,088 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:11,090 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-12-03T02:24:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=129 2024-12-03T02:24:11,092 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:11,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-03T02:24:11,094 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:11,094 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:11,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1580 sec 2024-12-03T02:24:11,100 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:11,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.2070 sec 2024-12-03T02:24:12,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T02:24:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-03T02:24:13,052 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:13,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-03T02:24:13,056 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:13,057 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:13,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:13,058 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-03T02:24:13,162 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:13,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=413 (was 409) - Thread LEAK? -, OpenFileDescriptor=704 (was 697) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=412 (was 448), ProcessCount=11 (was 11), AvailableMemoryMB=1735 (was 1791) 2024-12-03T02:24:13,196 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=413, OpenFileDescriptor=704, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=1734 2024-12-03T02:24:13,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:24:13,200 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,201 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x751cc797 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2069368a 2024-12-03T02:24:13,225 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x751cc797 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x751cc797 to 127.0.0.1:64747 2024-12-03T02:24:13,225 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:13,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:24:13,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132 2024-12-03T02:24:13,407 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:13,441 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:13,443 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-03T02:24:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-03T02:24:13,445 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:13,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-03T02:24:13,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-12-03T02:24:13,447 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:13,448 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 249 msec 2024-12-03T02:24:13,457 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,459 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fae5cb2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5623bd4a 2024-12-03T02:24:13,466 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:13,466 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-03T02:24:13,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:13,523 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:24:13,529 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,530 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5cbfc179 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79213081 2024-12-03T02:24:13,541 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x5cbfc179 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5cbfc179 to 127.0.0.1:64747 2024-12-03T02:24:13,541 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:13,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:24:13,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-03T02:24:13,732 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:13,735 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:13,736 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:13,736 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:13,736 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,739 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c27b386 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f68aa3f 2024-12-03T02:24:13,740 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:13,756 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:13,756 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:13,757 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-03T02:24:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-03T02:24:13,759 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:13,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-03T02:24:13,761 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:13,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-03T02:24:13,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 238 msec 2024-12-03T02:24:13,772 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,773 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x554a0829 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e10885b 2024-12-03T02:24:13,783 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:13,783 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-03T02:24:13,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:13,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:13,842 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:13,842 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:13,843 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:24:13,846 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:13,847 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0ad8b545 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11eee17c 2024-12-03T02:24:13,858 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0ad8b545 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0ad8b545 to 127.0.0.1:64747 2024-12-03T02:24:13,858 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:13,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:24:14,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-03T02:24:14,028 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:14,029 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:14,029 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:14,029 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:14,030 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,032 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cca3c50 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c7aa6f9 2024-12-03T02:24:14,032 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:14,048 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:14,048 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:14,048 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-03T02:24:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-03T02:24:14,051 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:14,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-03T02:24:14,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec 2024-12-03T02:24:14,053 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:14,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-03T02:24:14,063 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,064 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63e17321 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1e7a850d 2024-12-03T02:24:14,074 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:14,075 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-03T02:24:14,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:14,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-03T02:24:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:24:14,167 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,167 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62a63f8b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@520a6946 2024-12-03T02:24:14,175 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x62a63f8b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62a63f8b to 127.0.0.1:64747 2024-12-03T02:24:14,175 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:14,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:24:14,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-03T02:24:14,345 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:14,346 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:14,346 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:14,346 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:14,346 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,348 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x012a097e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18e529f0 2024-12-03T02:24:14,348 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:14,379 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:14,379 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:14,380 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-03T02:24:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-03T02:24:14,382 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:14,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-03T02:24:14,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-03T02:24:14,384 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-03T02:24:14,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec 2024-12-03T02:24:14,392 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,393 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51e8a6d4 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c3ef657 2024-12-03T02:24:14,399 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:14,400 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:14,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-03T02:24:14,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:14,482 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,483 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,486 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:14,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T02:24:14,491 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,492 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50a30263 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c8410ad 2024-12-03T02:24:14,500 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x50a30263 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50a30263 to 127.0.0.1:64747 2024-12-03T02:24:14,500 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:14,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T02:24:14,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-03T02:24:14,670 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:14,674 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:14,674 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:14,674 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:14,674 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,678 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6906affb to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4dc8153f 2024-12-03T02:24:14,678 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:14,697 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:14,697 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:14,697 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-03T02:24:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-03T02:24:14,700 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:14,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-03T02:24:14,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-03T02:24:14,701 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:14,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 213 msec 2024-12-03T02:24:14,710 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,711 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7484d1f3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b927504 2024-12-03T02:24:14,716 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:14,716 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-03T02:24:14,811 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:14,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:14,813 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-03T02:24:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T02:24:14,815 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:14,816 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66c56f11 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d79fb8b 2024-12-03T02:24:14,824 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x66c56f11 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66c56f11 to 127.0.0.1:64747 2024-12-03T02:24:14,824 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:14,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T02:24:14,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-03T02:24:14,995 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:14,997 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:14,997 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:14,998 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:14,998 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,000 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2592c853 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@351349db 2024-12-03T02:24:15,001 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:15,018 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:15,019 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:15,019 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-03T02:24:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-03T02:24:15,021 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:15,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-03T02:24:15,023 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-03T02:24:15,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-03T02:24:15,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-03T02:24:15,032 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,032 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x248c31bf to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@712e0912 2024-12-03T02:24:15,041 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:15,041 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-03T02:24:15,131 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:15,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:15,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T02:24:15,135 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,136 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16a6743f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32d7df3 2024-12-03T02:24:15,141 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x16a6743f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x16a6743f to 127.0.0.1:64747 2024-12-03T02:24:15,142 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:15,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T02:24:15,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-03T02:24:15,312 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:15,314 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:15,314 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:15,314 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:15,314 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,316 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ec9a3b0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@251d1727 2024-12-03T02:24:15,317 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:15,331 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:15,331 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:15,332 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-03T02:24:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-03T02:24:15,334 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:15,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-03T02:24:15,336 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:15,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 176 msec 2024-12-03T02:24:15,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 204 msec 2024-12-03T02:24:15,344 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,345 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x520d5faf to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29052aa7 2024-12-03T02:24:15,357 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:15,357 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-03T02:24:15,451 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:15,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:15,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:15,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T02:24:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:15,455 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:15,456 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:15,456 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(648): PEER '1', shared lock count=0, exclusively locked by procId=146 2024-12-03T02:24:15,456 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:15,456 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:15,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:15,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:15,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:15,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:15,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:15,467 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T02:24:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:15,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147 2024-12-03T02:24:15,620 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:15,620 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:15,620 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:15,620 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:15,621 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,623 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71be5995 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@719d92c2 2024-12-03T02:24:15,623 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:15,624 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:15,627 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-03T02:24:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-03T02:24:15,630 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:15,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-03T02:24:15,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-03T02:24:15,632 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:15,632 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:15,636 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:15,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-03T02:24:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-03T02:24:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:15,770 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:15,770 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:15,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T02:24:15,773 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:15,774 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=148, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:15,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=148, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:15,774 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-03T02:24:15,881 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:15,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=456 (was 413) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x554a0829-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x520d5faf-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x012a097e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x248c31bf-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6ec9a3b0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7484d1f3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cca3c50 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x554a0829-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3fae5cb2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2592c853 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6ec9a3b0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2c27b386 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e17321 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e17321-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7484d1f3-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3fae5cb2-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x51e8a6d4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6906affb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6906affb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7484d1f3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x012a097e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x71be5995-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e17321-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x71be5995-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6906affb-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6ec9a3b0-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2c27b386-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3fae5cb2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cca3c50-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x248c31bf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x520d5faf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cca3c50-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x51e8a6d4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2c27b386-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x71be5995 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x554a0829 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2592c853-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x248c31bf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x51e8a6d4-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2592c853-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x012a097e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x520d5faf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=759 (was 704) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=1655 (was 1734) 2024-12-03T02:24:15,908 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=456, OpenFileDescriptor=759, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=1654 2024-12-03T02:24:15,909 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T02:24:15,912 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:15,913 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x238b73fd to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c0ab248 2024-12-03T02:24:15,925 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x238b73fd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x238b73fd to 127.0.0.1:64747 2024-12-03T02:24:15,925 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:15,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T02:24:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:16,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=150 2024-12-03T02:24:16,128 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:16,165 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:16,169 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-12-03T02:24:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-12-03T02:24:16,172 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:16,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-03T02:24:16,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-03T02:24:16,175 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:16,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 266 msec 2024-12-03T02:24:16,188 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,190 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a2eaad9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c3f7257 2024-12-03T02:24:16,200 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:16,200 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-03T02:24:16,231 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:16,231 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T02:24:16,235 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,236 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e9f6c19 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d752ced 2024-12-03T02:24:16,261 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x2e9f6c19 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e9f6c19 to 127.0.0.1:64747 2024-12-03T02:24:16,261 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:16,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T02:24:16,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-03T02:24:16,431 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:16,432 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:16,433 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:16,433 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:16,433 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,434 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3591dd8b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@120e9b8e 2024-12-03T02:24:16,434 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:16,449 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:16,450 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:16,450 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-03T02:24:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-12-03T02:24:16,452 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:16,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-03T02:24:16,454 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:16,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-03T02:24:16,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 222 msec 2024-12-03T02:24:16,469 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,470 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17dfcf15 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@677f9668 2024-12-03T02:24:16,483 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:16,483 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-03T02:24:16,550 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:16,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:24:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-03T02:24:16,556 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,557 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3234cb9b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@220e252d 2024-12-03T02:24:16,566 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3234cb9b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3234cb9b to 127.0.0.1:64747 2024-12-03T02:24:16,566 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:16,590 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:16,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-03T02:24:16,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-03T02:24:16,776 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:16,777 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:16,777 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:16,777 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:16,777 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,779 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b829ec5 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@442938c6 2024-12-03T02:24:16,780 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:16,801 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:16,801 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:16,801 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-03T02:24:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=154 2024-12-03T02:24:16,804 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:16,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-03T02:24:16,806 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:24:16,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-03T02:24:16,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 254 msec 2024-12-03T02:24:16,814 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,816 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x122be27f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@191bf44a 2024-12-03T02:24:16,824 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:16,825 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-03T02:24:16,870 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:16,871 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-03T02:24:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-03T02:24:16,874 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,876 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f01cd09 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@615f4488 2024-12-03T02:24:16,883 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x2f01cd09 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f01cd09 to 127.0.0.1:64747 2024-12-03T02:24:16,883 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:16,883 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:16,885 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=155, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:16,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=155, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=14 msec 2024-12-03T02:24:16,886 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-03T02:24:16,980 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-03T02:24:16,981 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:24:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:24:16,983 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:16,984 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0941096b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3cc2c8c0 2024-12-03T02:24:16,991 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0941096b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0941096b to 127.0.0.1:64747 2024-12-03T02:24:16,991 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:17,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:24:17,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=157 2024-12-03T02:24:17,161 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:17,162 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:17,162 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:17,162 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:17,163 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:17,164 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2796b1c4 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@31ae2099 2024-12-03T02:24:17,164 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:17,181 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:17,181 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:17,181 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-03T02:24:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-03T02:24:17,185 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:17,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-03T02:24:17,190 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-03T02:24:17,190 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:24:17,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 209 msec 2024-12-03T02:24:17,197 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:17,198 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f954016 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3884a733 2024-12-03T02:24:17,207 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:17,208 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-03T02:24:17,301 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:17,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-03T02:24:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:24:17,308 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:17,309 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d7ff8d4 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b835ee 2024-12-03T02:24:17,316 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x2d7ff8d4 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d7ff8d4 to 127.0.0.1:64747 2024-12-03T02:24:17,316 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:17,317 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:17,318 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=158, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:17,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=158, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=14 msec 2024-12-03T02:24:17,319 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-03T02:24:17,411 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-03T02:24:17,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-03T02:24:17,416 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:17,416 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:17,417 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:17,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:17,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:17,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:17,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:17,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:17,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-03T02:24:17,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=160 2024-12-03T02:24:17,580 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:17,580 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:17,580 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:17,580 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:17,580 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:17,581 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31355914 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34fccf6c 2024-12-03T02:24:17,582 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:17,582 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:17,584 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160 2024-12-03T02:24:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=160 2024-12-03T02:24:17,587 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:17,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-03T02:24:17,589 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:17,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec 2024-12-03T02:24:17,589 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:17,597 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:17,600 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 186 msec 2024-12-03T02:24:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-03T02:24:17,731 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:17,732 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-03T02:24:17,734 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:17,736 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=161, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:17,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=161, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:17,737 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-03T02:24:17,841 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:17,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:17,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:24:17,843 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:17,844 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=162, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:17,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=162, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:17,845 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-03T02:24:17,951 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:17,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=480 (was 456) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2796b1c4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6a2eaad9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4b829ec5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3591dd8b-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2f954016-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6a2eaad9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6a2eaad9-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x122be27f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3591dd8b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4b829ec5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2f954016 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2796b1c4-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31355914-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x17dfcf15-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x17dfcf15 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x17dfcf15-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x4b829ec5-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x122be27f-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3591dd8b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31355914-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2796b1c4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x122be27f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2f954016-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31355914 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=791 (was 759) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=1612 (was 1654) 2024-12-03T02:24:17,989 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=480, OpenFileDescriptor=791, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=1612 2024-12-03T02:24:17,990 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-03T02:24:17,993 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:17,994 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0215c97c to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68b4bc34 2024-12-03T02:24:18,016 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0215c97c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0215c97c to 127.0.0.1:64747 2024-12-03T02:24:18,017 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:18,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-03T02:24:18,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=164 2024-12-03T02:24:18,194 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:18,221 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:18,223 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164 2024-12-03T02:24:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=164 2024-12-03T02:24:18,225 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:18,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-03T02:24:18,228 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:18,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-03T02:24:18,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 238 msec 2024-12-03T02:24:18,237 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:18,238 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5650e4e0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@316bf7bb 2024-12-03T02:24:18,249 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:18,249 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-03T02:24:18,311 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:18,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-03T02:24:18,316 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:18,317 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03b692e3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57aab2ba 2024-12-03T02:24:18,324 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x03b692e3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x03b692e3 to 127.0.0.1:64747 2024-12-03T02:24:18,324 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:18,325 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:18,326 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=165, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:18,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=165, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=14 msec 2024-12-03T02:24:18,327 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-03T02:24:18,421 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-03T02:24:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:18,423 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:24:18,425 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:18,427 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=166, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:18,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=166, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:18,428 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-03T02:24:18,530 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:18,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:18,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-03T02:24:18,534 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:24:18,535 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3aec81f1 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d72da2b 2024-12-03T02:24:18,541 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3aec81f1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3aec81f1 to 127.0.0.1:64747 2024-12-03T02:24:18,541 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:18,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-03T02:24:18,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=168 2024-12-03T02:24:18,720 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-03T02:24:18,757 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.ReplicationSource(231): queueId=2-64035cbb0b46,40081,1733192624478, ReplicationSource: 2, currentBandwidth=0 2024-12-03T02:24:18,760 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=168 2024-12-03T02:24:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=168 2024-12-03T02:24:18,762 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:18,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-03T02:24:18,764 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:18,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec 2024-12-03T02:24:18,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 233 msec 2024-12-03T02:24:18,772 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:24:18,773 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a4431d4 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@258426fc 2024-12-03T02:24:18,782 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:18,783 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-03T02:24:18,851 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-03T02:24:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:18,853 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-03T02:24:18,857 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:18,858 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:18,858 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:18,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:18,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:18,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:18,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:18,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:18,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-03T02:24:19,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-03T02:24:19,021 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:19,021 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:19,022 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:19,022 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:19,022 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:19,025 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52ccad44 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b8eb31a 2024-12-03T02:24:19,026 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:19,027 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:19,029 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-03T02:24:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=170 2024-12-03T02:24:19,031 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:19,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-03T02:24:19,033 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:19,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-03T02:24:19,033 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:19,038 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:19,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-03T02:24:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-03T02:24:19,170 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:19,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-03T02:24:19,175 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-03T02:24:19,175 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-03T02:24:19,175 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-03T02:24:19,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-03T02:24:19,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-03T02:24:19,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-03T02:24:19,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-03T02:24:19,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-03T02:24:19,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-03T02:24:19,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-03T02:24:19,337 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-03T02:24:19,337 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-03T02:24:19,337 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:19,338 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-03T02:24:19,338 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test2' 2024-12-03T02:24:19,340 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63136a42 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3acffe87 2024-12-03T02:24:19,341 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,2-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:19,341 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(562): Done with the queue 2-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:19,345 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-03T02:24:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=172 2024-12-03T02:24:19,348 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:19,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-03T02:24:19,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 166 msec 2024-12-03T02:24:19,352 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-03T02:24:19,353 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-03T02:24:19,359 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-03T02:24:19,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 188 msec 2024-12-03T02:24:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-03T02:24:19,492 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-03T02:24:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:19,495 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-03T02:24:19,498 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:19,499 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=173, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:19,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=173, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:19,499 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-03T02:24:19,600 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:19,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:19,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T02:24:19,604 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:19,605 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=174, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:19,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=174, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:19,606 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:19,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-03T02:24:19,711 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:19,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=492 (was 480) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5650e4e0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x52ccad44-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x52ccad44-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63136a42 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5650e4e0-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4431d4-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5650e4e0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x52ccad44 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63136a42-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63136a42-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4431d4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5a4431d4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=807 (was 791) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=1612 (was 1612) 2024-12-03T02:24:19,746 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=492, OpenFileDescriptor=807, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=1611 2024-12-03T02:24:19,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-03T02:24:19,750 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-03T02:24:19,752 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=175, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:19,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=175, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec 2024-12-03T02:24:19,753 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-03T02:24:19,861 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-03T02:24:19,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T02:24:19,864 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:19,865 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=176, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:19,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=176, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:19,866 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-03T02:24:19,970 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:19,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-03T02:24:19,973 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:19,974 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=177, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:19,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=177, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:19,975 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-03T02:24:20,080 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:20,096 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=492 (was 492), OpenFileDescriptor=805 (was 807), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=1611 (was 1611) 2024-12-03T02:24:20,108 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=492, OpenFileDescriptor=805, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=1611 2024-12-03T02:24:20,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T02:24:20,110 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-03T02:24:20,111 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=178, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:20,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=178, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=2 msec 2024-12-03T02:24:20,112 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-03T02:24:20,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-03T02:24:20,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:20,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:20,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-03T02:24:20,223 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:20,224 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=179, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:20,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=179, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:20,225 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:20,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-03T02:24:20,330 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:20,330 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T02:24:20,332 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:20,334 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=180, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:20,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=180, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:20,335 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-03T02:24:20,441 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:20,466 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=492 (was 492), OpenFileDescriptor=805 (was 805), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=1627 (was 1611) - AvailableMemoryMB LEAK? - 2024-12-03T02:24:20,479 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=492, OpenFileDescriptor=805, MaxFileDescriptor=1048576, SystemLoadAverage=387, ProcessCount=11, AvailableMemoryMB=1627 2024-12-03T02:24:20,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-03T02:24:20,483 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:20,484 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x692d2780 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2be46911 2024-12-03T02:24:20,491 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x692d2780 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x692d2780 to 127.0.0.1:64747 2024-12-03T02:24:20,491 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:20,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-03T02:24:20,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=182 2024-12-03T02:24:20,702 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:20,726 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:20,729 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=182 2024-12-03T02:24:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=182 2024-12-03T02:24:20,730 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:20,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-12-03T02:24:20,732 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:20,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-03T02:24:20,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 252 msec 2024-12-03T02:24:20,741 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:20,742 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x647675a2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@463ed991 2024-12-03T02:24:20,749 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:20,749 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-03T02:24:20,800 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:20,801 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:20,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-03T02:24:20,804 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:20,805 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42b82139 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6b5ce4d9 2024-12-03T02:24:20,816 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x42b82139 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x42b82139 to 127.0.0.1:64747 2024-12-03T02:24:20,816 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:20,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-03T02:24:20,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-03T02:24:20,986 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:20,988 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:20,988 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:20,988 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:20,988 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:20,989 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2098c5f3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6369b2b3 2024-12-03T02:24:20,989 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:21,003 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:21,003 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:21,004 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-03T02:24:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=184 2024-12-03T02:24:21,006 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:21,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-12-03T02:24:21,008 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:21,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-03T02:24:21,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 207 msec 2024-12-03T02:24:21,020 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,022 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26d38d08 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7239cf90 2024-12-03T02:24:21,032 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:21,033 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:21,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-03T02:24:21,121 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:21,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-03T02:24:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-03T02:24:21,125 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,126 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1cfe8c1a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27ebe201 2024-12-03T02:24:21,141 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x1cfe8c1a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1cfe8c1a to 127.0.0.1:64747 2024-12-03T02:24:21,141 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:21,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-03T02:24:21,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-03T02:24:21,304 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:21,305 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:21,305 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:21,305 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:21,305 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,306 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3bd8f289 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2781cf0e 2024-12-03T02:24:21,307 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:21,323 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:21,323 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:21,327 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-03T02:24:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=186 2024-12-03T02:24:21,330 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:21,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=186, resume processing ppid=185 2024-12-03T02:24:21,332 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-03T02:24:21,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, ppid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-03T02:24:21,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-03T02:24:21,351 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,353 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31dc5038 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7066c76e 2024-12-03T02:24:21,366 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:21,366 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-03T02:24:21,440 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:21,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:21,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:24:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-03T02:24:21,446 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,448 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6fd8ac6d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55bce3b3 2024-12-03T02:24:21,457 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x6fd8ac6d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6fd8ac6d to 127.0.0.1:64747 2024-12-03T02:24:21,459 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:21,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-03T02:24:21,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-03T02:24:21,628 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:21,629 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:21,629 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:21,629 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:21,630 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,631 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63d4de66 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d540358 2024-12-03T02:24:21,632 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:21,652 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:21,652 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:21,652 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-03T02:24:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=188 2024-12-03T02:24:21,657 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:21,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=188, resume processing ppid=187 2024-12-03T02:24:21,659 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-03T02:24:21,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, ppid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-03T02:24:21,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 217 msec 2024-12-03T02:24:21,669 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,671 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7565f295 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@301ccf58 2024-12-03T02:24:21,682 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:21,683 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-03T02:24:21,760 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:21,760 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:21,761 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-03T02:24:21,765 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:21,765 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:21,766 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:21,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:21,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:21,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:21,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:21,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:21,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-03T02:24:21,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-03T02:24:21,928 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:21,928 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:21,928 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:21,928 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:21,928 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:21,930 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63e8f201 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@114d46d8 2024-12-03T02:24:21,930 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:21,931 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:21,934 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-03T02:24:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=190 2024-12-03T02:24:21,937 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:21,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=190, resume processing ppid=189 2024-12-03T02:24:21,940 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:21,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, ppid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-03T02:24:21,940 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:21,949 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:21,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 188 msec 2024-12-03T02:24:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-03T02:24:22,081 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:22,081 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-03T02:24:22,084 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:22,085 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=191, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:22,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=191, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-03T02:24:22,087 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-03T02:24:22,191 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:22,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=192, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-03T02:24:22,194 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:22,195 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=192, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:22,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=192, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:22,196 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-03T02:24:22,300 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:22,325 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=516 (was 492) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26d38d08-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e8f201 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63d4de66-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31dc5038-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x647675a2-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31dc5038-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26d38d08 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26d38d08-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63d4de66 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7565f295-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3bd8f289-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2098c5f3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x31dc5038 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e8f201-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x647675a2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63e8f201-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3bd8f289-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2098c5f3-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3bd8f289 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7565f295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2098c5f3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x63d4de66-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7565f295-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x647675a2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=837 (was 805) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 387), ProcessCount=11 (was 11), AvailableMemoryMB=2639 (was 1627) - AvailableMemoryMB LEAK? - 2024-12-03T02:24:22,327 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=516 is superior to 500 2024-12-03T02:24:22,344 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=516, OpenFileDescriptor=837, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=2636 2024-12-03T02:24:22,344 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=516 is superior to 500 2024-12-03T02:24:22,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-03T02:24:22,350 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,351 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0701190a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b8f0aa1 2024-12-03T02:24:22,366 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x0701190a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0701190a to 127.0.0.1:64747 2024-12-03T02:24:22,366 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:22,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=194, ppid=193, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-03T02:24:22,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=194 2024-12-03T02:24:22,577 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:22,606 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:22,609 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=194 2024-12-03T02:24:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=194 2024-12-03T02:24:22,612 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:22,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=194, resume processing ppid=193 2024-12-03T02:24:22,614 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:22,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=194, ppid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-03T02:24:22,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 269 msec 2024-12-03T02:24:22,625 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,626 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x723f432e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@112098ca 2024-12-03T02:24:22,632 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:22,633 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-03T02:24:22,670 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:22,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-03T02:24:22,674 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,674 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a9f9bda to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76b6ba48 2024-12-03T02:24:22,702 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x3a9f9bda {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a9f9bda to 127.0.0.1:64747 2024-12-03T02:24:22,702 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-03T02:24:22,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:22,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-03T02:24:22,936 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:22,937 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:22,937 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:22,937 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:22,937 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,938 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68272c06 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54e598f 2024-12-03T02:24:22,939 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:22,957 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:22,958 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:22,958 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-03T02:24:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=196 2024-12-03T02:24:22,960 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:22,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=196, resume processing ppid=195 2024-12-03T02:24:22,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=196, ppid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 177 msec 2024-12-03T02:24:22,962 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-03T02:24:22,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 291 msec 2024-12-03T02:24:22,972 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,974 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69d5f8e8 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d72971d 2024-12-03T02:24:22,982 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:22,983 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-03T02:24:22,991 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:22,991 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:22,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-03T02:24:22,995 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:22,996 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23d74c3f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@200f6b2f 2024-12-03T02:24:23,007 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x23d74c3f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x23d74c3f to 127.0.0.1:64747 2024-12-03T02:24:23,007 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:23,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:23,084 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-03T02:24:23,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-03T02:24:23,177 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:23,178 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:23,178 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:23,178 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:23,178 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,179 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d00b4ac to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b81fcb1 2024-12-03T02:24:23,180 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:23,193 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:23,193 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:23,194 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-03T02:24:23,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=198 2024-12-03T02:24:23,196 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:23,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=198, resume processing ppid=197 2024-12-03T02:24:23,197 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-03T02:24:23,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=198, ppid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 172 msec 2024-12-03T02:24:23,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 205 msec 2024-12-03T02:24:23,207 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,207 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2190c2f7 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67141c6f 2024-12-03T02:24:23,268 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:23,268 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-03T02:24:23,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:23,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:23,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:23,312 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-03T02:24:23,315 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,316 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e532e50 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b6e31aa 2024-12-03T02:24:23,332 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x6e532e50 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e532e50 to 127.0.0.1:64747 2024-12-03T02:24:23,332 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:23,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-03T02:24:23,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-03T02:24:23,503 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:23,504 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:23,504 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:23,505 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:23,505 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,506 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ca70c9d to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@397c79b3 2024-12-03T02:24:23,507 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:23,528 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:23,528 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:23,528 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-03T02:24:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=200 2024-12-03T02:24:23,536 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:23,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=200, resume processing ppid=199 2024-12-03T02:24:23,538 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:23,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=200, ppid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 186 msec 2024-12-03T02:24:23,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 225 msec 2024-12-03T02:24:23,549 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,550 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6812b540 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23492cde 2024-12-03T02:24:23,566 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:23,566 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-03T02:24:23,632 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:23,633 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:23,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:23,635 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-03T02:24:23,644 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,645 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5fab481f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@554da45f 2024-12-03T02:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:23,707 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x5fab481f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5fab481f to 127.0.0.1:64747 2024-12-03T02:24:23,708 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:23,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-03T02:24:23,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-03T02:24:23,883 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:23,885 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:23,885 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:23,886 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:23,886 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,888 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6547f11e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ece978c 2024-12-03T02:24:23,889 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:23,911 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:23,911 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:23,915 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-03T02:24:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=202 2024-12-03T02:24:23,930 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:23,932 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d8bfab9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24e5bc8a 2024-12-03T02:24:23,939 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-03T02:24:23,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=202, resume processing ppid=201 2024-12-03T02:24:23,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=202, ppid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 210 msec 2024-12-03T02:24:23,951 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-03T02:24:23,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 317 msec 2024-12-03T02:24:23,963 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:23,964 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-03T02:24:24,261 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:24,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:24,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:24,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-03T02:24:24,268 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,269 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x384ad2a3 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63702ae2 2024-12-03T02:24:24,300 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x384ad2a3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x384ad2a3 to 127.0.0.1:64747 2024-12-03T02:24:24,301 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:24,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-03T02:24:24,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-03T02:24:24,469 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:24,471 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:24,471 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:24,471 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:24,471 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,473 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13c36995 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6184ca6 2024-12-03T02:24:24,474 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:24,501 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:24,501 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:24,502 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-03T02:24:24,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=204 2024-12-03T02:24:24,505 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:24,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=204, resume processing ppid=203 2024-12-03T02:24:24,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=204, ppid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-03T02:24:24,507 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:24,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 243 msec 2024-12-03T02:24:24,520 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,521 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x196285a0 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@712363a8 2024-12-03T02:24:24,540 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:24,541 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-03T02:24:24,580 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:24,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:24,582 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:24,583 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-03T02:24:24,587 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,588 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11671bb6 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ba115a2 2024-12-03T02:24:24,607 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x11671bb6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x11671bb6 to 127.0.0.1:64747 2024-12-03T02:24:24,607 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:24,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-03T02:24:24,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-03T02:24:24,778 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:24,782 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:24,782 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:24,782 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:24,782 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,784 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x751605d2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ab651b6 2024-12-03T02:24:24,784 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:24,806 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:24,807 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:24,807 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-03T02:24:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=206 2024-12-03T02:24:24,820 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:24,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=206, resume processing ppid=205 2024-12-03T02:24:24,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=206, ppid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec 2024-12-03T02:24:24,824 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:24,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 240 msec 2024-12-03T02:24:24,830 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,831 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26c26b7a to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4faafe3f 2024-12-03T02:24:24,862 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:24,863 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-03T02:24:24,907 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:24,908 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:24,910 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-03T02:24:24,916 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:24,917 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76453153 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@674a57c0 2024-12-03T02:24:24,925 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x76453153 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x76453153 to 127.0.0.1:64747 2024-12-03T02:24:24,925 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:24,950 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-03T02:24:25,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-03T02:24:25,110 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:25,111 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-03T02:24:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=208 2024-12-03T02:24:25,115 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:25,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=208, resume processing ppid=207 2024-12-03T02:24:25,118 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-03T02:24:25,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=208, ppid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-03T02:24:25,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 208 msec 2024-12-03T02:24:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-03T02:24:25,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:25,230 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:25,231 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:25,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-03T02:24:25,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-03T02:24:25,235 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,235 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x184d5573 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3655421f 2024-12-03T02:24:25,274 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x184d5573 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x184d5573 to 127.0.0.1:64747 2024-12-03T02:24:25,274 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:25,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-03T02:24:25,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-03T02:24:25,444 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:25,446 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:25,446 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:25,446 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:25,446 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,448 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x762bb38c to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45ff8bf6 2024-12-03T02:24:25,448 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:25,470 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:25,471 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:25,471 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-03T02:24:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=210 2024-12-03T02:24:25,473 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:25,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=210, resume processing ppid=209 2024-12-03T02:24:25,475 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-03T02:24:25,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=210, ppid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-03T02:24:25,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 243 msec 2024-12-03T02:24:25,490 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,492 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2548d633 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1268224 2024-12-03T02:24:25,511 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:25,511 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-03T02:24:25,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:25,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:25,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-03T02:24:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-03T02:24:25,564 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,565 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x52d183d2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2de8d23a 2024-12-03T02:24:25,583 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x52d183d2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x52d183d2 to 127.0.0.1:64747 2024-12-03T02:24:25,584 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:25,593 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-03T02:24:25,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-03T02:24:25,747 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:25,748 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:25,749 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:25,749 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:25,749 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,750 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a7ec334 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@302eb3b2 2024-12-03T02:24:25,751 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:25,772 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:25,773 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:25,775 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-03T02:24:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=212 2024-12-03T02:24:25,777 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:25,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=212, resume processing ppid=211 2024-12-03T02:24:25,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=212, ppid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-03T02:24:25,780 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-03T02:24:25,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec 2024-12-03T02:24:25,790 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:25,791 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50f91dc4 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55cbf6a8 2024-12-03T02:24:25,848 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:25,848 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-03T02:24:25,880 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:25,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:25,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-03T02:24:25,886 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:25,886 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:25,887 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:25,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:25,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:25,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:25,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:25,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:25,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-03T02:24:26,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-03T02:24:26,093 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:26,093 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:26,093 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:26,094 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:26,094 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:26,095 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cacbe6f to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@466eb1c4 2024-12-03T02:24:26,095 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:26,095 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:26,098 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-03T02:24:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=214 2024-12-03T02:24:26,135 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:26,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=214, resume processing ppid=213 2024-12-03T02:24:26,137 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:26,137 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:26,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=214, ppid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-12-03T02:24:26,142 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:26,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 260 msec 2024-12-03T02:24:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-03T02:24:26,200 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:26,200 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-03T02:24:26,203 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:26,204 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=215, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:26,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=215, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:26,205 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-03T02:24:26,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:26,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=216, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-03T02:24:26,312 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:26,313 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=216, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:26,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=216, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:26,313 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-03T02:24:26,420 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:26,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=568 (was 516) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d8bfab9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x762bb38c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5ca70c9d-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a7ec334 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x762bb38c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x762bb38c-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x196285a0-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2190c2f7-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x50f91dc4-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x723f432e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2548d633-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cacbe6f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2548d633-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cacbe6f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d00b4ac java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a7ec334-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69d5f8e8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x50f91dc4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x751605d2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x196285a0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69d5f8e8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x68272c06 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x68272c06-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5ca70c9d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2190c2f7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6812b540-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d8bfab9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6812b540 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x751605d2-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d8bfab9-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x196285a0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13c36995-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6547f11e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6547f11e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x69d5f8e8-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x751605d2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6cacbe6f-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6547f11e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x68272c06-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26c26b7a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x50f91dc4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6812b540-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13c36995-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x723f432e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26c26b7a-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d00b4ac-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0a7ec334-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0d00b4ac-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2190c2f7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x5ca70c9d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x26c26b7a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x723f432e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x13c36995 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2548d633 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=907 (was 837) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3051 (was 2636) - AvailableMemoryMB LEAK? - 2024-12-03T02:24:26,438 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=568 is superior to 500 2024-12-03T02:24:26,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=568, OpenFileDescriptor=907, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=3049 2024-12-03T02:24:26,450 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=568 is superior to 500 2024-12-03T02:24:26,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-03T02:24:26,453 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:26,454 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69daf0c9 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a95bb7e 2024-12-03T02:24:26,509 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x69daf0c9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69daf0c9 to 127.0.0.1:64747 2024-12-03T02:24:26,509 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-03T02:24:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-03T02:24:26,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=218, ppid=217, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:26,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=218 2024-12-03T02:24:26,934 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:26,964 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:26,967 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=218 2024-12-03T02:24:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=218 2024-12-03T02:24:26,970 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:26,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=218, resume processing ppid=217 2024-12-03T02:24:26,972 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:26,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=218, ppid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-03T02:24:26,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 521 msec 2024-12-03T02:24:26,982 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:26,983 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1117315b to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70fa7c01 2024-12-03T02:24:27,047 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:27,047 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-03T02:24:27,080 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:27,081 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3992): Client=jenkins//172.17.0.3 disable replication peer, id=1 2024-12-03T02:24:27,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-03T02:24:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-03T02:24:27,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-03T02:24:27,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-03T02:24:27,285 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-03T02:24:27,286 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-03T02:24:27,286 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-03T02:24:27,286 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:27,286 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:27,288 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3328b224 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cdfcd2e 2024-12-03T02:24:27,288 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:27,312 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:27,312 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-03T02:24:27,313 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-03T02:24:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=220 2024-12-03T02:24:27,322 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:27,332 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:27,333 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x393d9f4e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@790d0d76 2024-12-03T02:24:27,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=220, resume processing ppid=219 2024-12-03T02:24:27,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=220, ppid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-03T02:24:27,335 INFO [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-03T02:24:27,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 254 msec 2024-12-03T02:24:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-03T02:24:27,400 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:27,401 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-03T02:24:27,433 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:27,433 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:27,433 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:27,434 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:27,434 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:27,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:27,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:27,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:27,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:27,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:27,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-03T02:24:27,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-03T02:24:27,630 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:27,630 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:27,630 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:27,630 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:27,630 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:27,632 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39d78fe6 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58cdc22e 2024-12-03T02:24:27,632 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:27,632 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:27,634 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-03T02:24:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=222 2024-12-03T02:24:27,636 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:27,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=222, resume processing ppid=221 2024-12-03T02:24:27,638 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:27,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=222, ppid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 158 msec 2024-12-03T02:24:27,638 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:27,643 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:27,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 242 msec 2024-12-03T02:24:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-03T02:24:27,721 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:27,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-03T02:24:27,723 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:27,724 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=223, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:27,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=223, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:27,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-03T02:24:27,830 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:27,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=224, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-03T02:24:27,833 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:27,834 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=224, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:27,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=224, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:27,835 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-03T02:24:27,940 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:27,965 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=579 (was 568) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3328b224 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x393d9f4e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x39d78fe6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x39d78fe6-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x393d9f4e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3328b224-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x393d9f4e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1117315b-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1117315b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x39d78fe6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1117315b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x3328b224-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=921 (was 907) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=2805 (was 3049) 2024-12-03T02:24:27,965 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-03T02:24:27,977 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=579, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=2803 2024-12-03T02:24:27,978 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-03T02:24:27,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=dummy_peer 2024-12-03T02:24:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:27,981 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-03T02:24:27,981 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=225, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:27,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=225, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:27,982 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:28,081 INFO [master/64035cbb0b46:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T02:24:28,081 INFO [master/64035cbb0b46:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T02:24:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-03T02:24:28,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-03T02:24:28,091 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-03T02:24:28,093 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:28,094 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=226, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:28,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=226, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:28,094 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-03T02:24:28,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:28,202 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-03T02:24:28,204 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:28,205 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=227, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:28,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=227, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:28,206 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-03T02:24:28,311 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:28,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=579 (was 579), OpenFileDescriptor=921 (was 921), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=2790 (was 2803) 2024-12-03T02:24:28,333 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-03T02:24:28,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=579, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=2787 2024-12-03T02:24:28,349 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=579 is superior to 500 2024-12-03T02:24:28,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-03T02:24:28,354 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:28,356 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01f754b2 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ebc9396 2024-12-03T02:24:28,384 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x01f754b2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01f754b2 to 127.0.0.1:64747 2024-12-03T02:24:28,384 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:28,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=229, ppid=228, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-03T02:24:28,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=229 2024-12-03T02:24:28,573 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:28,614 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:28,617 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=229 2024-12-03T02:24:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=229 2024-12-03T02:24:28,619 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:28,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=229, resume processing ppid=228 2024-12-03T02:24:28,622 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:28,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=229, ppid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 203 msec 2024-12-03T02:24:28,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 271 msec 2024-12-03T02:24:28,637 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:28,638 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b09d5da to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f04ca0a 2024-12-03T02:24:28,666 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:28,666 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-03T02:24:28,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4025): Client=jenkins//172.17.0.3 list replication peers, regex=null 2024-12-03T02:24:28,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-03T02:24:28,684 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:28,684 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:28,685 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:28,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:28,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-03T02:24:28,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-03T02:24:28,893 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:28,893 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:28,894 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:28,894 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:28,894 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:28,895 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d68bf21 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44d41d06 2024-12-03T02:24:28,895 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:28,895 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:28,897 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-03T02:24:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=231 2024-12-03T02:24:28,899 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:28,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=231, resume processing ppid=230 2024-12-03T02:24:28,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=231, ppid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-03T02:24:28,900 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:28,900 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:28,905 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:28,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 229 msec 2024-12-03T02:24:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-03T02:24:29,000 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:29,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-03T02:24:29,002 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,003 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=232, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:29,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=232, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-03T02:24:29,004 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-03T02:24:29,110 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:29,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=233, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-03T02:24:29,112 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,113 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=233, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:29,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=233, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-03T02:24:29,114 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-03T02:24:29,221 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:29,244 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=585 (was 579) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6d68bf21-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7b09d5da-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7b09d5da-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6d68bf21 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x7b09d5da java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x6d68bf21-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=929 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=2630 (was 2787) 2024-12-03T02:24:29,244 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2024-12-03T02:24:29,258 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=585, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=2627 2024-12-03T02:24:29,258 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2024-12-03T02:24:29,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.CallRunner(138): callId: 690 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:39128 deadline: 1733192729261, exception=java.io.IOException: Replication peer modification disabled 2024-12-03T02:24:29,262 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:24:29,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] ipc.CallRunner(138): callId: 691 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.3:39128 deadline: 1733192729370, exception=java.io.IOException: Replication peer modification disabled 2024-12-03T02:24:29,371 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 110 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-03T02:24:29,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-03T02:24:29.262Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:24:29.371Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-03T02:24:29,374 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-03T02:24:29,376 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,377 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=234, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:29,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=234, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-03T02:24:29,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-03T02:24:29,480 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:29,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-03T02:24:29,482 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:29,483 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=235, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:29,484 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=235, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-03T02:24:29,484 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-03T02:24:29,591 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:29,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=585 (was 585), OpenFileDescriptor=929 (was 929), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=2565 (was 2627) 2024-12-03T02:24:29,610 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2024-12-03T02:24:29,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=585, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=2557 2024-12-03T02:24:29,624 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=585 is superior to 500 2024-12-03T02:24:29,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-03T02:24:29,628 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:29,630 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65bb9538 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69cb2204 2024-12-03T02:24:29,641 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x65bb9538 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x65bb9538 to 127.0.0.1:64747 2024-12-03T02:24:29,641 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:29,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=237, ppid=236, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-03T02:24:29,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=237 2024-12-03T02:24:29,819 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:29,856 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:29,859 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=237 2024-12-03T02:24:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=237 2024-12-03T02:24:29,861 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:29,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=237, resume processing ppid=236 2024-12-03T02:24:29,864 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:29,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=237, ppid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-03T02:24:29,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 239 msec 2024-12-03T02:24:29,875 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:29,876 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b4dafd7 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@629ad011 2024-12-03T02:24:29,882 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:29,882 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-03T02:24:29,940 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:29,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:29,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-03T02:24:29,945 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:29,945 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:29,946 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:29,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:29,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:29,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:29,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:29,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:29,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-03T02:24:30,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-03T02:24:30,110 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:30,110 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:30,110 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:30,110 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:30,110 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:30,111 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1621be12 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69a4577f 2024-12-03T02:24:30,112 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:30,112 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:30,114 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-03T02:24:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=239 2024-12-03T02:24:30,116 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:30,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=239, resume processing ppid=238 2024-12-03T02:24:30,118 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:30,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=239, ppid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-03T02:24:30,118 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:30,125 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:30,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 183 msec 2024-12-03T02:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-03T02:24:30,260 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:30,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-03T02:24:30,263 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:30,264 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=240, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:30,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=240, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:30,265 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-03T02:24:30,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:30,389 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=591 (was 585) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b4dafd7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b4dafd7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x0b4dafd7-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1621be12-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1621be12 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1621be12-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=937 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=2481 (was 2557) 2024-12-03T02:24:30,389 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2024-12-03T02:24:30,403 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=591, OpenFileDescriptor=937, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=2480 2024-12-03T02:24:30,403 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=591 is superior to 500 2024-12-03T02:24:30,404 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=241, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-03T02:24:30,407 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:30,407 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x292d2d72 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6428dea8 2024-12-03T02:24:30,415 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x292d2d72 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x292d2d72 to 127.0.0.1:64747 2024-12-03T02:24:30,416 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:30,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=242, ppid=241, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-03T02:24:30,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=242 2024-12-03T02:24:30,656 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:30,693 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:30,696 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=242 2024-12-03T02:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=242 2024-12-03T02:24:30,698 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:30,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=242, resume processing ppid=241 2024-12-03T02:24:30,706 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:30,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=242, ppid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-03T02:24:30,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 302 msec 2024-12-03T02:24:30,716 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:30,718 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2eb90e08 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a73bdc2 2024-12-03T02:24:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-03T02:24:30,727 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:30,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:30,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4014): Client=jenkins//172.17.0.3 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-03T02:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-03T02:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-03T02:24:30,732 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:30,733 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x553c89c1 to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@dd38286 2024-12-03T02:24:30,749 DEBUG [ReadOnlyZKClient-127.0.0.1:64747@0x553c89c1 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x553c89c1 to 127.0.0.1:64747 2024-12-03T02:24:30,749 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T02:24:30,750 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-03T02:24:30,750 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-03T02:24:30,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-03T02:24:30,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-03T02:24:30,919 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-03T02:24:30,920 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-03T02:24:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=244 2024-12-03T02:24:30,922 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:30,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=244, resume processing ppid=243 2024-12-03T02:24:30,924 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:64747/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-03T02:24:30,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=244, ppid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-12-03T02:24:30,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 195 msec 2024-12-03T02:24:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-03T02:24:31,060 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-03T02:24:31,061 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4002): Client=jenkins//172.17.0.3 get replication peer config, id=1 2024-12-03T02:24:31,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-03T02:24:31,065 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:31,065 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:31,065 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:31,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:31,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:31,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:31,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:31,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:31,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-03T02:24:31,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-03T02:24:31,266 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:31,266 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:31,266 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:31,266 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-03T02:24:31,267 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:64747' and parent='/hbase-test1' 2024-12-03T02:24:31,268 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1700717e to 127.0.0.1:64747 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24dae3af 2024-12-03T02:24:31,269 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-03T02:24:31,269 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:31,271 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-03T02:24:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=246 2024-12-03T02:24:31,273 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:31,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=246, resume processing ppid=245 2024-12-03T02:24:31,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=246, ppid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-03T02:24:31,275 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:31,275 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:31,280 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:31,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 218 msec 2024-12-03T02:24:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-03T02:24:31,380 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:31,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:31,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:31,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247 2024-12-03T02:24:31,384 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:31,385 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=247, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:31,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=247, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:31,386 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247 2024-12-03T02:24:31,491 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-03T02:24:31,491 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:31,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=248, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248 2024-12-03T02:24:31,494 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:31,495 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=248, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:31,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=248, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-03T02:24:31,497 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:31,597 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248 2024-12-03T02:24:31,601 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:31,631 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=597 (was 591) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1700717e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1700717e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2eb90e08-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x1700717e-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2eb90e08-SendThread(127.0.0.1:64747) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:64747@0x2eb90e08 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$857/0x00007fef44be4ff8.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=945 (was 937) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=467 (was 438) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2386 (was 2480) 2024-12-03T02:24:31,631 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2024-12-03T02:24:31,647 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=597, OpenFileDescriptor=945, MaxFileDescriptor=1048576, SystemLoadAverage=467, ProcessCount=11, AvailableMemoryMB=2384 2024-12-03T02:24:31,647 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2024-12-03T02:24:31,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-03T02:24:31,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=250, ppid=249, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-03T02:24:31,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=250 2024-12-03T02:24:31,836 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-03T02:24:31,863 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.ReplicationSource(231): queueId=1-64035cbb0b46,40081,1733192624478, ReplicationSource: 1, currentBandwidth=0 2024-12-03T02:24:31,865 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=250 2024-12-03T02:24:31,865 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-64035cbb0b46,40081,1733192624478 (queues=1) is replicating from cluster=5b5d6b8d-1904-4383-be12-bce3422a64f5 to cluster=5b5d6b8d-1904-4383-be12-bce3422a64f5 2024-12-03T02:24:31,865 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=64035cbb0b46%2C40081%2C1733192624478 2024-12-03T02:24:31,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=250 2024-12-03T02:24:31,865 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-64035cbb0b46,40081,1733192624478, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-03T02:24:31,866 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 64035cbb0b46%2C40081%2C1733192624478 2024-12-03T02:24:31,866 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.wal-reader.64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.1733192626668, startPosition=0, beingWritten=true 2024-12-03T02:24:31,867 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:31,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=250, resume processing ppid=249 2024-12-03T02:24:31,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=250, ppid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-03T02:24:31,868 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-03T02:24:31,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 219 msec 2024-12-03T02:24:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-03T02:24:31,972 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:31,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3973): Client=jenkins//172.17.0.3 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-03T02:24:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-03T02:24:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-03T02:24:31,979 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-03T02:24:31,980 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=251, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:31,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=251, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=7 msec 2024-12-03T02:24:31,981 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-03T02:24:32,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-03T02:24:32,092 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=1 2024-12-03T02:24:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=252, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:32,097 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:32,097 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:32,098 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:32,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-03T02:24:32,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:32,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-03T02:24:32,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-03T02:24:32,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-03T02:24:32,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=253, ppid=252, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-03T02:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:32,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40081 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=253 2024-12-03T02:24:32,261 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-03T02:24:32,261 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-03T02:24:32,261 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-64035cbb0b46,40081,1733192624478 because: Replication stream was removed by a user 2024-12-03T02:24:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:33,262 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-03T02:24:33,262 WARN [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.wal-reader.64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-03T02:24:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-03T02:24:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:34,262 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0.replicationSource,1-64035cbb0b46,40081,1733192624478.replicationSource.shipper64035cbb0b46%2C40081%2C1733192624478,1-64035cbb0b46,40081,1733192624478 terminated 2024-12-03T02:24:34,262 INFO [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(562): Done with the queue 1-64035cbb0b46,40081,1733192624478 2024-12-03T02:24:34,264 DEBUG [RS_REFRESH_PEER-regionserver/64035cbb0b46:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=253 2024-12-03T02:24:34,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(4169): Remote procedure done, pid=253 2024-12-03T02:24:34,267 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on 64035cbb0b46,40081,1733192624478 suceeded 2024-12-03T02:24:34,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=253, resume processing ppid=252 2024-12-03T02:24:34,269 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-03T02:24:34,269 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-03T02:24:34,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=253, ppid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1590 sec 2024-12-03T02:24:34,275 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-03T02:24:34,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1830 sec 2024-12-03T02:24:36,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-03T02:24:36,260 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-03T02:24:36,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.HMaster(3980): Client=jenkins//172.17.0.3 removing replication peer, id=2 2024-12-03T02:24:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] procedure2.ProcedureExecutor(1139): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-03T02:24:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-03T02:24:36,264 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-03T02:24:36,269 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=254, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-03T02:24:36,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=254, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-03T02:24:36,270 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-03T02:24:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39895 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-03T02:24:36,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-03T02:24:36,394 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=598 (was 597) - Thread LEAK? -, OpenFileDescriptor=950 (was 945) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 467), ProcessCount=11 (was 11), AvailableMemoryMB=2074 (was 2384) 2024-12-03T02:24:36,395 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2024-12-03T02:24:36,395 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:24:36,395 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:24:36,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:24:36,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T02:24:36,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:24:36,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:24:36,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T02:24:36,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T02:24:36,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=195587028, stopped=false 2024-12-03T02:24:36,397 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=64035cbb0b46,39895,1733192623709 2024-12-03T02:24:36,414 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:24:36,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T02:24:36,415 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:24:36,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:24:36,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:24:36,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T02:24:36,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:24:36,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:24:36,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T02:24:36,416 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '64035cbb0b46,40081,1733192624478' ***** 2024-12-03T02:24:36,416 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T02:24:36,416 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T02:24:36,416 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T02:24:36,416 INFO [RS:0;64035cbb0b46:40081 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T02:24:36,417 INFO [RS:0;64035cbb0b46:40081 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T02:24:36,417 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(3091): Received CLOSE for badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:36,417 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(959): stopping server 64035cbb0b46,40081,1733192624478 2024-12-03T02:24:36,417 INFO [RS:0;64035cbb0b46:40081 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:24:36,417 INFO [RS:0;64035cbb0b46:40081 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;64035cbb0b46:40081. 2024-12-03T02:24:36,417 DEBUG [RS:0;64035cbb0b46:40081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T02:24:36,418 DEBUG [RS:0;64035cbb0b46:40081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:36,418 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T02:24:36,418 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T02:24:36,418 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T02:24:36,418 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing badaa07e05f1c98a805a3234eb1add78, disabling compactions & flushes 2024-12-03T02:24:36,418 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. after waiting 0 ms 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:24:36,418 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T02:24:36,418 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, badaa07e05f1c98a805a3234eb1add78=hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78.} 2024-12-03T02:24:36,418 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T02:24:36,418 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T02:24:36,419 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:36,422 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-12-03T02:24:36,422 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing badaa07e05f1c98a805a3234eb1add78 3/3 column families, dataSize=5.05 KB heapSize=10.72 KB 2024-12-03T02:24:36,483 INFO [regionserver/64035cbb0b46:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:24:36,489 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/info/53d935e6eeb44a8f842298082ad26fc0 is 147, key is hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78./info:regioninfo/1733192629203/Put/seqid=0 2024-12-03T02:24:36,489 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/hfileref/227b053036bb419b9b87e0158a6aa4b5 is 33, key is 1/hfileref:/1733192674274/DeleteFamily/seqid=0 2024-12-03T02:24:36,520 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T02:24:36,520 INFO [regionserver/64035cbb0b46:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T02:24:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741839_1015 (size=5733) 2024-12-03T02:24:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741838_1014 (size=6631) 2024-12-03T02:24:36,529 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/info/53d935e6eeb44a8f842298082ad26fc0 2024-12-03T02:24:36,594 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/ns/ecace14e5555493f9c2b73007c3483ba is 43, key is default/ns:d/1733192627933/Put/seqid=0 2024-12-03T02:24:36,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741840_1016 (size=5153) 2024-12-03T02:24:36,619 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:36,771 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T02:24:36,819 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:36,929 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=638 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/hfileref/227b053036bb419b9b87e0158a6aa4b5 2024-12-03T02:24:36,946 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 227b053036bb419b9b87e0158a6aa4b5 2024-12-03T02:24:36,961 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/queue/ac09d395ba714f8286e1397df767cfc0 is 63, key is 1-64035cbb0b46,40081,1733192624478/queue:/1733192674263/DeleteFamily/seqid=0 2024-12-03T02:24:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741841_1017 (size=6582) 2024-12-03T02:24:37,005 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/ns/ecace14e5555493f9c2b73007c3483ba 2024-12-03T02:24:37,019 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:37,032 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/table/8ceb3381865b4079994fc6a0f59993a5 is 53, key is hbase:replication/table:state/1733192629219/Put/seqid=0 2024-12-03T02:24:37,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741842_1018 (size=5256) 2024-12-03T02:24:37,220 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, badaa07e05f1c98a805a3234eb1add78 2024-12-03T02:24:37,368 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.43 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/queue/ac09d395ba714f8286e1397df767cfc0 2024-12-03T02:24:37,376 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ac09d395ba714f8286e1397df767cfc0 2024-12-03T02:24:37,378 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/hfileref/227b053036bb419b9b87e0158a6aa4b5 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/hfileref/227b053036bb419b9b87e0158a6aa4b5 2024-12-03T02:24:37,385 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 227b053036bb419b9b87e0158a6aa4b5 2024-12-03T02:24:37,386 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/hfileref/227b053036bb419b9b87e0158a6aa4b5, entries=22, sequenceid=70, filesize=5.6 K 2024-12-03T02:24:37,388 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/.tmp/queue/ac09d395ba714f8286e1397df767cfc0 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/queue/ac09d395ba714f8286e1397df767cfc0 2024-12-03T02:24:37,397 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ac09d395ba714f8286e1397df767cfc0 2024-12-03T02:24:37,397 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/queue/ac09d395ba714f8286e1397df767cfc0, entries=22, sequenceid=70, filesize=6.4 K 2024-12-03T02:24:37,402 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for badaa07e05f1c98a805a3234eb1add78 in 979ms, sequenceid=70, compaction requested=false 2024-12-03T02:24:37,410 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/replication/badaa07e05f1c98a805a3234eb1add78/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1 2024-12-03T02:24:37,413 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:24:37,413 INFO [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:24:37,413 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for badaa07e05f1c98a805a3234eb1add78: Waiting for close lock at 1733192676417Running coprocessor pre-close hooks at 1733192676418 (+1 ms)Disabling compacts and flushes for region at 1733192676418Disabling writes for close at 1733192676418Obtaining lock to block concurrent updates at 1733192676422 (+4 ms)Preparing flush snapshotting stores in badaa07e05f1c98a805a3234eb1add78 at 1733192676422Finished memstore snapshotting hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78., syncing WAL and waiting on mvcc, flushsize=dataSize=5170, getHeapSize=10928, getOffHeapSize=0, getCellsCount=66 at 1733192676430 (+8 ms)Flushing stores of hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. at 1733192676431 (+1 ms)Flushing badaa07e05f1c98a805a3234eb1add78/hfileref: creating writer at 1733192676434 (+3 ms)Flushing badaa07e05f1c98a805a3234eb1add78/hfileref: appending metadata at 1733192676476 (+42 ms)Flushing badaa07e05f1c98a805a3234eb1add78/hfileref: closing flushed file at 1733192676478 (+2 ms)Flushing badaa07e05f1c98a805a3234eb1add78/queue: creating writer at 1733192676946 (+468 ms)Flushing badaa07e05f1c98a805a3234eb1add78/queue: appending metadata at 1733192676960 (+14 ms)Flushing badaa07e05f1c98a805a3234eb1add78/queue: closing flushed file at 1733192676960Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26237f0d: reopening flushed file at 1733192677376 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5abc0f5e: reopening flushed file at 1733192677387 (+11 ms)Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for badaa07e05f1c98a805a3234eb1add78 in 979ms, sequenceid=70, compaction requested=false at 1733192677402 (+15 ms)Writing region close event to WAL at 1733192677405 (+3 ms)Running coprocessor post-close hooks at 1733192677411 (+6 ms)Closed at 1733192677413 (+2 ms) 2024-12-03T02:24:37,414 DEBUG [RS_CLOSE_REGION-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:replication,,1733192628360.badaa07e05f1c98a805a3234eb1add78. 2024-12-03T02:24:37,420 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T02:24:37,420 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T02:24:37,420 DEBUG [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T02:24:37,440 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/table/8ceb3381865b4079994fc6a0f59993a5 2024-12-03T02:24:37,453 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/info/53d935e6eeb44a8f842298082ad26fc0 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/info/53d935e6eeb44a8f842298082ad26fc0 2024-12-03T02:24:37,463 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/info/53d935e6eeb44a8f842298082ad26fc0, entries=10, sequenceid=11, filesize=6.5 K 2024-12-03T02:24:37,465 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/ns/ecace14e5555493f9c2b73007c3483ba as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/ns/ecace14e5555493f9c2b73007c3483ba 2024-12-03T02:24:37,474 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/ns/ecace14e5555493f9c2b73007c3483ba, entries=2, sequenceid=11, filesize=5.0 K 2024-12-03T02:24:37,476 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/.tmp/table/8ceb3381865b4079994fc6a0f59993a5 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/table/8ceb3381865b4079994fc6a0f59993a5 2024-12-03T02:24:37,485 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/table/8ceb3381865b4079994fc6a0f59993a5, entries=2, sequenceid=11, filesize=5.1 K 2024-12-03T02:24:37,486 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1067ms, sequenceid=11, compaction requested=false 2024-12-03T02:24:37,492 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-03T02:24:37,493 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T02:24:37,493 INFO [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T02:24:37,493 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733192676418Running coprocessor pre-close hooks at 1733192676418Disabling compacts and flushes for region at 1733192676418Disabling writes for close at 1733192676418Obtaining lock to block concurrent updates at 1733192676422 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1733192676422Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1368, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1733192676430 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733192676431 (+1 ms)Flushing 1588230740/info: creating writer at 1733192676434 (+3 ms)Flushing 1588230740/info: appending metadata at 1733192676476 (+42 ms)Flushing 1588230740/info: closing flushed file at 1733192676478 (+2 ms)Flushing 1588230740/ns: creating writer at 1733192676579 (+101 ms)Flushing 1588230740/ns: appending metadata at 1733192676594 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733192676594Flushing 1588230740/table: creating writer at 1733192677017 (+423 ms)Flushing 1588230740/table: appending metadata at 1733192677031 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733192677031Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ee8ba2d: reopening flushed file at 1733192677452 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aadd4fb: reopening flushed file at 1733192677463 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dea92cb: reopening flushed file at 1733192677474 (+11 ms)Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1067ms, sequenceid=11, compaction requested=false at 1733192677486 (+12 ms)Writing region close event to WAL at 1733192677488 (+2 ms)Running coprocessor post-close hooks at 1733192677493 (+5 ms)Closed at 1733192677493 2024-12-03T02:24:37,493 DEBUG [RS_CLOSE_META-regionserver/64035cbb0b46:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T02:24:37,621 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(976): stopping server 64035cbb0b46,40081,1733192624478; all regions closed. 2024-12-03T02:24:37,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741834_1010 (size=2742) 2024-12-03T02:24:37,636 DEBUG [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs 2024-12-03T02:24:37,636 INFO [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C40081%2C1733192624478.meta:.meta(num 1733192627670) 2024-12-03T02:24:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741837_1013 (size=11128) 2024-12-03T02:24:37,639 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.rep.1733192629152 not finished, retry = 0 2024-12-03T02:24:37,743 DEBUG [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs 2024-12-03T02:24:37,743 INFO [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C40081%2C1733192624478.rep:(num 1733192629152) 2024-12-03T02:24:37,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741832_1008 (size=93) 2024-12-03T02:24:37,746 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/WALs/64035cbb0b46,40081,1733192624478/64035cbb0b46%2C40081%2C1733192624478.1733192626668 not finished, retry = 0 2024-12-03T02:24:37,850 DEBUG [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/oldWALs 2024-12-03T02:24:37,850 INFO [RS:0;64035cbb0b46:40081 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 64035cbb0b46%2C40081%2C1733192624478:(num 1733192626668) 2024-12-03T02:24:37,850 DEBUG [RS:0;64035cbb0b46:40081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T02:24:37,850 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T02:24:37,850 INFO [RS:0;64035cbb0b46:40081 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:24:37,850 INFO [RS:0;64035cbb0b46:40081 {}] hbase.ChoreService(370): Chore service for: regionserver/64035cbb0b46:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T02:24:37,851 INFO [regionserver/64035cbb0b46:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:24:37,851 INFO [RS:0;64035cbb0b46:40081 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:24:37,852 INFO [RS:0;64035cbb0b46:40081 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40081 2024-12-03T02:24:37,864 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/64035cbb0b46,40081,1733192624478 2024-12-03T02:24:37,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T02:24:37,864 INFO [RS:0;64035cbb0b46:40081 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:24:37,873 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [64035cbb0b46,40081,1733192624478] 2024-12-03T02:24:37,881 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/64035cbb0b46,40081,1733192624478 already deleted, retry=false 2024-12-03T02:24:37,881 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 64035cbb0b46,40081,1733192624478 expired; onlineServers=0 2024-12-03T02:24:37,881 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '64035cbb0b46,39895,1733192623709' ***** 2024-12-03T02:24:37,881 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T02:24:37,881 INFO [M:0;64035cbb0b46:39895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T02:24:37,881 INFO [M:0;64035cbb0b46:39895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T02:24:37,882 DEBUG [M:0;64035cbb0b46:39895 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T02:24:37,882 DEBUG [M:0;64035cbb0b46:39895 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T02:24:37,882 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T02:24:37,882 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192626377 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.large.0-1733192626377,5,FailOnTimeoutGroup] 2024-12-03T02:24:37,882 DEBUG [master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192626378 {}] cleaner.HFileCleaner(306): Exit Thread[master/64035cbb0b46:0:becomeActiveMaster-HFileCleaner.small.0-1733192626378,5,FailOnTimeoutGroup] 2024-12-03T02:24:37,882 INFO [M:0;64035cbb0b46:39895 {}] hbase.ChoreService(370): Chore service for: master/64035cbb0b46:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T02:24:37,882 INFO [M:0;64035cbb0b46:39895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T02:24:37,882 DEBUG [M:0;64035cbb0b46:39895 {}] master.HMaster(1795): Stopping service threads 2024-12-03T02:24:37,882 INFO [M:0;64035cbb0b46:39895 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T02:24:37,883 INFO [M:0;64035cbb0b46:39895 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T02:24:37,883 INFO [M:0;64035cbb0b46:39895 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T02:24:37,883 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T02:24:37,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T02:24:37,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T02:24:37,890 DEBUG [M:0;64035cbb0b46:39895 {}] zookeeper.ZKUtil(347): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T02:24:37,890 WARN [M:0;64035cbb0b46:39895 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T02:24:37,893 INFO [M:0;64035cbb0b46:39895 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/.lastflushedseqids 2024-12-03T02:24:37,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741843_1019 (size=172) 2024-12-03T02:24:37,910 INFO [M:0;64035cbb0b46:39895 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T02:24:37,910 INFO [M:0;64035cbb0b46:39895 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T02:24:37,911 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T02:24:37,911 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:24:37,911 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:24:37,911 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T02:24:37,911 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:24:37,911 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=748.89 KB heapSize=894.39 KB 2024-12-03T02:24:37,933 DEBUG [M:0;64035cbb0b46:39895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6552bf3a170f4432a0cc1f747babe9d3 is 82, key is hbase:meta,,1/info:regioninfo/1733192627794/Put/seqid=0 2024-12-03T02:24:37,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741844_1020 (size=5672) 2024-12-03T02:24:37,973 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:24:37,973 INFO [RS:0;64035cbb0b46:40081 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:24:37,973 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40081-0x1019a4a54f20001, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:24:37,973 INFO [RS:0;64035cbb0b46:40081 {}] regionserver.HRegionServer(1031): Exiting; stopping=64035cbb0b46,40081,1733192624478; zookeeper connection closed. 2024-12-03T02:24:37,973 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@220f24d9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@220f24d9 2024-12-03T02:24:37,974 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T02:24:38,343 INFO [M:0;64035cbb0b46:39895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1766 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6552bf3a170f4432a0cc1f747babe9d3 2024-12-03T02:24:38,381 DEBUG [M:0;64035cbb0b46:39895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e90e8d36f107436daf12331b96ebd415 is 2004, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1733192656886/Put/seqid=0 2024-12-03T02:24:38,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741845_1021 (size=162088) 2024-12-03T02:24:38,784 INFO [M:0;64035cbb0b46:39895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=746.34 KB at sequenceid=1766 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e90e8d36f107436daf12331b96ebd415 2024-12-03T02:24:38,791 INFO [M:0;64035cbb0b46:39895 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e90e8d36f107436daf12331b96ebd415 2024-12-03T02:24:38,806 DEBUG [M:0;64035cbb0b46:39895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/924b3472a0ec4819808423683427d1fd is 69, key is 64035cbb0b46,40081,1733192624478/rs:state/1733192626430/Put/seqid=0 2024-12-03T02:24:38,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741846_1022 (size=5156) 2024-12-03T02:24:39,214 INFO [M:0;64035cbb0b46:39895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1766 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/924b3472a0ec4819808423683427d1fd 2024-12-03T02:24:39,243 DEBUG [M:0;64035cbb0b46:39895 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/938a069cc57c4d5c887b5cf161741153 is 68, key is replication_peer_modification_on/state:d/1733192676374/Put/seqid=0 2024-12-03T02:24:39,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741847_1023 (size=5154) 2024-12-03T02:24:39,648 INFO [M:0;64035cbb0b46:39895 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1766 (bloomFilter=true), to=hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/938a069cc57c4d5c887b5cf161741153 2024-12-03T02:24:39,656 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6552bf3a170f4432a0cc1f747babe9d3 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6552bf3a170f4432a0cc1f747babe9d3 2024-12-03T02:24:39,664 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6552bf3a170f4432a0cc1f747babe9d3, entries=8, sequenceid=1766, filesize=5.5 K 2024-12-03T02:24:39,666 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e90e8d36f107436daf12331b96ebd415 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e90e8d36f107436daf12331b96ebd415 2024-12-03T02:24:39,679 INFO [M:0;64035cbb0b46:39895 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e90e8d36f107436daf12331b96ebd415 2024-12-03T02:24:39,679 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e90e8d36f107436daf12331b96ebd415, entries=254, sequenceid=1766, filesize=158.3 K 2024-12-03T02:24:39,681 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/924b3472a0ec4819808423683427d1fd as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/924b3472a0ec4819808423683427d1fd 2024-12-03T02:24:39,695 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/924b3472a0ec4819808423683427d1fd, entries=1, sequenceid=1766, filesize=5.0 K 2024-12-03T02:24:39,699 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/938a069cc57c4d5c887b5cf161741153 as hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/938a069cc57c4d5c887b5cf161741153 2024-12-03T02:24:39,712 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44743/user/jenkins/test-data/caf674f8-26b6-7b48-727a-86f878e96731/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/938a069cc57c4d5c887b5cf161741153, entries=1, sequenceid=1766, filesize=5.0 K 2024-12-03T02:24:39,714 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(3140): Finished flush of dataSize ~748.89 KB/766867, heapSize ~894.33 KB/915792, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1803ms, sequenceid=1766, compaction requested=false 2024-12-03T02:24:39,717 INFO [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T02:24:39,717 DEBUG [M:0;64035cbb0b46:39895 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733192677910Disabling compacts and flushes for region at 1733192677910Disabling writes for close at 1733192677911 (+1 ms)Obtaining lock to block concurrent updates at 1733192677911Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733192677911Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=766867, getHeapSize=915792, getOffHeapSize=0, getCellsCount=1957 at 1733192677911Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733192677912 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733192677912Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733192677932 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733192677932Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733192678354 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733192678380 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733192678380Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733192678791 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733192678806 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733192678806Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733192679227 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733192679242 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733192679242Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b4c76e: reopening flushed file at 1733192679655 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c2a8555: reopening flushed file at 1733192679664 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@680f2e2: reopening flushed file at 1733192679679 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b4ccd64: reopening flushed file at 1733192679695 (+16 ms)Finished flush of dataSize ~748.89 KB/766867, heapSize ~894.33 KB/915792, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1803ms, sequenceid=1766, compaction requested=false at 1733192679714 (+19 ms)Writing region close event to WAL at 1733192679717 (+3 ms)Closed at 1733192679717 2024-12-03T02:24:39,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33433 is added to blk_1073741830_1006 (size=882874) 2024-12-03T02:24:39,726 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T02:24:39,727 INFO [M:0;64035cbb0b46:39895 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T02:24:39,727 INFO [M:0;64035cbb0b46:39895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39895 2024-12-03T02:24:39,727 INFO [M:0;64035cbb0b46:39895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T02:24:39,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:24:39,839 INFO [M:0;64035cbb0b46:39895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T02:24:39,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39895-0x1019a4a54f20000, quorum=127.0.0.1:64747, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T02:24:39,844 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e80f2cb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T02:24:39,846 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43176244{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:24:39,846 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:24:39,846 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62b38b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:24:39,847 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@698dfd8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.log.dir/,STOPPED} 2024-12-03T02:24:39,856 WARN [BP-1640616088-172.17.0.3-1733192619772 heartbeating to localhost/127.0.0.1:44743 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T02:24:39,856 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T02:24:39,856 WARN [BP-1640616088-172.17.0.3-1733192619772 heartbeating to localhost/127.0.0.1:44743 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1640616088-172.17.0.3-1733192619772 (Datanode Uuid c613eedb-8ecc-4dc8-9c59-b16125dd16b5) service to localhost/127.0.0.1:44743 2024-12-03T02:24:39,856 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T02:24:39,858 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/data/data1/current/BP-1640616088-172.17.0.3-1733192619772 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:24:39,858 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/cluster_a4401341-840f-6e5b-5bc8-3316c75dc587/data/data2/current/BP-1640616088-172.17.0.3-1733192619772 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T02:24:39,859 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T02:24:39,866 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1441d181{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T02:24:39,866 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T02:24:39,866 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T02:24:39,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T02:24:39,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1970dd8a-c1fc-3ac5-4513-19fb6db6b391/hadoop.log.dir/,STOPPED} 2024-12-03T02:24:39,888 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T02:24:39,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down